/** * V2ProjectSyncStrategy - Handles synchronization in newo_v2 format * * Uses the SAME V1 API endpoints as ProjectSyncStrategy but writes/reads * files in the newo_v2 directory layout: * {CustomerIdn}/ * import_version.txt * {ProjectIdn}/ * {project_idn}.yaml * agents/{AgentIdn}/ * agent.yaml * flows/{FlowIdn}/ * {FlowIdn}.yaml (inline skill defs, events, state_fields) * skills/{SkillIdn}.nsl|.nslg */ import type { ISyncStrategy, PullOptions, PullResult, PushResult, ChangeItem, ValidationResult, ValidationError, StatusSummary } from './ISyncStrategy.js'; import type { CustomerConfig, ILogger, HashStore } from '../../resources/common/types.js'; import type { AxiosInstance } from 'axios'; import type { ProjectMeta, Agent, Flow, Skill, FlowEvent, FlowState, FlowMetadata, ProjectData, ProjectMap, SkillMetadata } from '../../../types.js'; import type { LocalProjectData, LocalAgentData, LocalFlowData, LocalSkillData, ApiClientFactory } from './ProjectSyncStrategy.js'; import fs from 'fs-extra'; import path from 'path'; import { listProjects, listAgents, listFlowSkills, listFlowEvents, listFlowStates, createSkill, createSkillParameter, updateSkill, publishFlow, getProjectAttributes, getCustomerAttributes, listLibraries, updateLibrarySkill, getFlow, } from '../../../api.js'; import { syncFlowMetadata, emptyFlowSyncCounts, totalFlowSyncOps, describeFlowSyncCounts } from '../../../sync/flow-metadata.js'; import type { LibraryResponse } from '../../../api.js'; import { ensureStateOnly, writeFileSafe, mapPath, } from '../../../fsutil.js'; import { sha256, saveHashes, loadHashes } from '../../../hash.js'; import { v2ImportVersionPath, v2ProjectYamlPath, v2AgentDir, v2AgentYamlPath, v2FlowYamlPath, v2SkillScriptPath, v2SkillRelativePath, v2ProjectAttributesPath, v2CustomerAttributesPath, v2AkbDir, v2AkbPath, v2LibraryYamlPath, v2LibrarySkillScriptPath, v2LibrarySkillRelativePath, } from '../../../format/paths-v2.js'; import { V2_IMPORT_VERSION, } from '../../../format/types.js'; import { generateV2FlowYaml, generateV2ProjectYaml, generateV2AgentYaml, parseV2FlowYaml, buildV2InlineSkill, buildV2FlowEvent, buildV2StateField, type V2InlineSkill, type V2FlowEvent, type V2StateField, } from '../../../format/v2-yaml.js'; import { isContentDifferent } from '../../../sync/skill-files.js'; import yaml from 'js-yaml'; import { patchYamlToPyyaml } from '../../../format/yaml-patch.js'; import type { RunnerType, SkillParameter } from '../../../types.js'; interface V2FlowSkillTarget { projectIdn: string; agentIdn: string; flowIdn: string; skillIdn: string; skillData: SkillMetadata | undefined; } /** * V2ProjectSyncStrategy - same API, newo_v2 file layout */ export class V2ProjectSyncStrategy implements ISyncStrategy { readonly resourceType = 'projects'; readonly displayName = 'Projects (newo_v2)'; constructor( private apiClientFactory: ApiClientFactory, private logger: ILogger ) {} // ────────────────────────────────────── // PULL // ────────────────────────────────────── async pull(customer: CustomerConfig, options: PullOptions = {}): Promise> { const client = await this.apiClientFactory(customer, options.verbose ?? false); const hashes: HashStore = {}; const projects: LocalProjectData[] = []; this.logger.verbose(`[newo_v2] Loading project list for customer ${customer.idn}...`); // Use V2 state init (no V1 projects/ dir) await ensureStateOnly(customer.idn); // Write import_version.txt marker const versionPath = v2ImportVersionPath(customer.idn); await writeFileSafe(versionPath, V2_IMPORT_VERSION); // Write V2 customer attributes: attributes.yaml (sorted, with !enum ValueType.X) try { const custAttrs = await getCustomerAttributes(client, true); const attrs = custAttrs.attributes || []; if (attrs.length > 0) { const attrYaml = formatV2AttributesYaml(attrs); const custAttrPath = v2CustomerAttributesPath(customer.idn); await writeFileSafe(custAttrPath, attrYaml); hashes[custAttrPath] = sha256(attrYaml); } } catch { this.logger.verbose(` Could not pull customer attributes`); } // Fetch projects from API (same V1 endpoints) const apiProjects = options.projectId ? [{ id: options.projectId, idn: 'unknown', title: 'Project' } as ProjectMeta] : await listProjects(client); if (apiProjects.length === 0) { this.logger.info(`No projects found for customer ${customer.idn}`); return { items: [], count: 0, hashes: {} }; } // Load existing map for reference let existingMap: ProjectMap = { projects: {} }; const mapFile = mapPath(customer.idn); if (await fs.pathExists(mapFile)) { try { const mapData = await fs.readJson(mapFile); if (mapData && typeof mapData === 'object' && 'projects' in mapData) { existingMap = mapData as ProjectMap; } } catch { // Start fresh } } // Count total skills for progress let totalSkills = 0; let processedSkills = 0; for (const project of apiProjects) { const agents = await listAgents(client, project.id); for (const agent of agents) { const flows = agent.flows || []; for (const flow of flows) { const skills = await listFlowSkills(client, flow.id); totalSkills += skills.length; } } } this.logger.verbose(`[newo_v2] Total skills to process: ${totalSkills}`); // Process each project for (const project of apiProjects) { this.logger.verbose(`[newo_v2] Processing project: ${project.title} (${project.idn})`); // Write V2 project YAML: {project_idn}.yaml // The API returns registry_idn (not registry) - map to V2 field name const projectYaml = generateV2ProjectYaml({ idn: project.idn, name: project.title || project.idn, version: (project as any).version || '1.0.0', description: project.description || '', is_auto_update_enabled: (project as any).is_auto_update_enabled ?? false, registry: (project as any).registry_idn || (project as any).registry || '', registry_item_idn: (project as any).registry_item_idn || '', }); const projectYamlPath = v2ProjectYamlPath(customer.idn, project.idn); await writeFileSafe(projectYamlPath, projectYaml); hashes[projectYamlPath] = sha256(projectYaml); const localProject: LocalProjectData = { projectId: project.id, projectIdn: project.idn, metadata: { id: project.id, idn: project.idn, title: project.title, description: project.description || '', created_at: project.created_at || '', updated_at: project.updated_at || '', }, agents: [] }; const agents = await listAgents(client, project.id); this.logger.verbose(` Found ${agents.length} agents in project ${project.title}`); const projectData: ProjectData = { projectId: project.id, projectIdn: project.idn, agents: {} }; // Process each agent for (const agent of agents) { const localAgent = await this.pullAgent( client, customer, project, agent, hashes, options, () => { processedSkills++; if (!options.verbose && totalSkills > 0) { if (processedSkills % 10 === 0 || processedSkills === totalSkills) { this.logger.progress(processedSkills, totalSkills, '[newo_v2] Processing skills'); } } } ); localProject.agents.push(localAgent); // Build project data for map projectData.agents[agent.idn] = { id: agent.id, flows: {} }; for (const flow of localAgent.flows) { projectData.agents[agent.idn]!.flows[flow.idn] = { id: flow.id, skills: {} }; for (const skill of flow.skills) { projectData.agents[agent.idn]!.flows[flow.idn]!.skills[skill.idn] = skill.metadata; } } } // Pull libraries for this project try { const libraries = await listLibraries(client, project.id); if (libraries.length > 0) { this.logger.verbose(` Found ${libraries.length} libraries in project ${project.idn}`); projectData.libraries = {}; for (const lib of libraries) { await this.pullLibrary(client, customer, project, lib, hashes, options); projectData.libraries[lib.idn] = { id: lib.id, skills: {} }; for (const skill of lib.skills) { projectData.libraries[lib.idn]!.skills[skill.idn] = { id: skill.id, idn: skill.idn, title: skill.title, runner_type: skill.runner_type, model: skill.model, parameters: [...skill.parameters], path: skill.path }; } } } } catch { this.logger.verbose(` Could not pull libraries for project ${project.idn}`); } // Write V2 project attributes: {project_idn}/attributes.yaml try { const projAttrs = await getProjectAttributes(client, project.id, true); const attrs = projAttrs.attributes || []; if (attrs.length > 0) { const attrYaml = formatV2AttributesYaml(attrs); const attrPath = v2ProjectAttributesPath(customer.idn, project.idn); await writeFileSafe(attrPath, attrYaml); hashes[attrPath] = sha256(attrYaml); } } catch { this.logger.verbose(` Could not pull attributes for project ${project.idn}`); } existingMap.projects[project.idn] = projectData; projects.push(localProject); } // Write AKB stub files for all agents: akb/{AgentIdn}.yaml // V2 format creates an empty [] file for every agent persona const akbDirPath = v2AkbDir(customer.idn); await fs.ensureDir(akbDirPath); for (const project of projects) { for (const agent of project.agents) { const akbFilePath = v2AkbPath(customer.idn, agent.idn); if (!(await fs.pathExists(akbFilePath))) { await writeFileSafe(akbFilePath, '[]\n'); } // Don't overwrite existing AKB files that may have content from AkbSyncStrategy } } // Save updated project map await writeFileSafe(mapFile, JSON.stringify(existingMap, null, 2)); // Save hashes await saveHashes(hashes, customer.idn); return { items: projects, count: projects.length, hashes }; } /** * Pull a single agent in V2 format */ private async pullAgent( client: AxiosInstance, customer: CustomerConfig, project: ProjectMeta, agent: Agent, hashes: HashStore, options: PullOptions, onSkillProcessed: () => void ): Promise { this.logger.verbose(` [newo_v2] Processing agent: ${agent.title} (${agent.idn})`); // Write V2 agent YAML: agents/{AgentIdn}/agent.yaml // Preserve exact API values (null title stays null, "" description stays "") const agentYaml = generateV2AgentYaml({ idn: agent.idn, title: agent.title ?? null, description: agent.description ?? null, }); const agentYamlFilePath = v2AgentYamlPath(customer.idn, project.idn, agent.idn); await writeFileSafe(agentYamlFilePath, agentYaml); hashes[agentYamlFilePath] = sha256(agentYaml); const localAgent: LocalAgentData = { id: agent.id, idn: agent.idn, metadata: { id: agent.id, idn: agent.idn, title: agent.title || '', description: agent.description || '', }, flows: [] }; const flows = agent.flows || []; this.logger.verbose(` Found ${flows.length} flows in agent ${agent.title}`); for (const flow of flows) { const localFlow = await this.pullFlow( client, customer, project, agent, flow, hashes, options, onSkillProcessed ); localAgent.flows.push(localFlow); } return localAgent; } /** * Pull a single flow in V2 format * * In V2, the flow YAML contains inline skill definitions, events, and state_fields. * Skills are written to flows/{FlowIdn}/skills/{SkillIdn}.nsl|.nslg */ private async pullFlow( client: AxiosInstance, customer: CustomerConfig, project: ProjectMeta, agent: Agent, flow: Flow, hashes: HashStore, options: PullOptions, onSkillProcessed: () => void ): Promise { this.logger.verbose(` [newo_v2] Processing flow: ${flow.title} (${flow.idn})`); // Get flow events and states const [events, states] = await Promise.all([ listFlowEvents(client, flow.id).catch(() => [] as FlowEvent[]), listFlowStates(client, flow.id).catch(() => [] as FlowState[]) ]); // Process skills const skills = await listFlowSkills(client, flow.id); this.logger.verbose(` Found ${skills.length} skills in flow ${flow.title}`); // Build V2 inline skill definitions const v2Skills: V2InlineSkill[] = []; const localFlow: LocalFlowData = { id: flow.id, idn: flow.idn, metadata: { id: flow.id, idn: flow.idn, title: flow.title, description: flow.description || '', default_runner_type: flow.default_runner_type, default_model: flow.default_model, events, state_fields: states }, skills: [] }; for (const skill of skills) { const localSkill = await this.pullSkill( customer, project, agent, flow, skill, hashes, options ); localFlow.skills.push(localSkill); onSkillProcessed(); // Build inline skill definition for flow YAML const relPath = v2SkillRelativePath(flow.idn, skill.idn, skill.runner_type); v2Skills.push(buildV2InlineSkill( skill.idn, skill.title || '', skill.runner_type, skill.model?.model_idn || flow.default_model?.model_idn || '', skill.model?.provider_idn || flow.default_model?.provider_idn || '', skill.parameters.map(p => ({ name: p.name, default_value: p.default_value ?? '', })), relPath )); } // Build V2 events const v2Events = events.map(e => buildV2FlowEvent( e.idn, e.skill_selector || 'skill_idn', e.skill_idn || null, e.state_idn || null, e.integration_idn || null, e.connector_idn || null, e.interrupt_mode || 'queue' )); // Build V2 state fields const v2States = states.map(s => buildV2StateField( s.idn, s.title || '', s.default_value ?? '', s.scope || 'user' )); // Write V2 flow YAML: flows/{FlowIdn}/{FlowIdn}.yaml const flowYaml = generateV2FlowYaml( flow.idn, flow.title || flow.idn, flow.description ?? null, flow.default_runner_type || 'guidance', flow.default_model?.provider_idn || '', flow.default_model?.model_idn || '', v2Skills, v2Events, v2States ); const flowYamlFilePath = v2FlowYamlPath(customer.idn, project.idn, agent.idn, flow.idn); await writeFileSafe(flowYamlFilePath, flowYaml); hashes[flowYamlFilePath] = sha256(flowYaml); return localFlow; } /** * Pull a single skill script in V2 format * * Script goes to: flows/{FlowIdn}/skills/{SkillIdn}.nsl|.nslg * No separate metadata.yaml - metadata is inline in the flow YAML */ private async pullSkill( customer: CustomerConfig, project: ProjectMeta, agent: Agent, flow: Flow, skill: Skill, hashes: HashStore, options: PullOptions ): Promise { this.logger.verbose(` [newo_v2] Processing skill: ${skill.title} (${skill.idn})`); const scriptContent = skill.prompt_script || ''; const targetPath = v2SkillScriptPath( customer.idn, project.idn, agent.idn, flow.idn, skill.idn, skill.runner_type ); // Check for existing file and handle overwrites let shouldWrite = true; if (await fs.pathExists(targetPath)) { const existingContent = await fs.readFile(targetPath, 'utf8'); if (!isContentDifferent(existingContent, scriptContent)) { shouldWrite = false; hashes[targetPath] = sha256(scriptContent); } else if (!options.silentOverwrite) { // In non-silent mode, we overwrite (interactive mode handled in CLI layer) shouldWrite = true; } } if (shouldWrite) { await writeFileSafe(targetPath, scriptContent); hashes[targetPath] = sha256(scriptContent); } const skillMeta: SkillMetadata = { id: skill.id, idn: skill.idn, title: skill.title, runner_type: skill.runner_type, model: skill.model, parameters: [...skill.parameters], path: skill.path }; return { id: skill.id, idn: skill.idn, metadata: skillMeta, scriptPath: targetPath, scriptContent }; } /** * Pull a library and its skills in V2 format * * Writes: * {project}/libraries/{lib}/{lib}.yaml (with inline skill list) * {project}/libraries/{lib}/skills/{skill}.nsl|.nslg */ private async pullLibrary( _client: AxiosInstance, customer: CustomerConfig, project: ProjectMeta, lib: LibraryResponse, hashes: HashStore, _options: PullOptions ): Promise { this.logger.verbose(` [newo_v2] Processing library: ${lib.idn} (${lib.skills.length} skills)`); // Build V2 inline skill definitions for library YAML const v2Skills: V2InlineSkill[] = []; for (const skill of lib.skills) { const relPath = v2LibrarySkillRelativePath(project.idn, lib.idn, skill.idn, skill.runner_type); v2Skills.push(buildV2InlineSkill( skill.idn, skill.title || '', skill.runner_type, skill.model?.model_idn || '', skill.model?.provider_idn || '', skill.parameters.map(p => ({ name: p.name, default_value: p.default_value ?? '', })), relPath )); } // Sort skills same as flows const { sortV2Skills, sortV2Parameters } = await import('../../../format/v2-yaml.js'); const sortedSkills = sortV2Skills(v2Skills).map(s => ({ ...s, parameters: sortV2Parameters(s.parameters), })); // Write library YAML: libraries/{lib}/{lib}.yaml const libDef = { library: { idn: lib.idn, skills: sortedSkills, } }; const libYaml = yaml.dump(libDef, { indent: 2, lineWidth: -1, noRefs: true, sortKeys: false }); const libYamlPath = v2LibraryYamlPath(customer.idn, project.idn, lib.idn); await writeFileSafe(libYamlPath, libYaml); hashes[libYamlPath] = sha256(libYaml); // Write skill scripts for (const skill of lib.skills) { const scriptContent = skill.prompt_script || ''; const scriptPath = v2LibrarySkillScriptPath( customer.idn, project.idn, lib.idn, skill.idn, skill.runner_type ); let shouldWrite = true; if (await fs.pathExists(scriptPath)) { const existing = await fs.readFile(scriptPath, 'utf8'); if (!isContentDifferent(existing, scriptContent)) { shouldWrite = false; hashes[scriptPath] = sha256(scriptContent); } } if (shouldWrite) { await writeFileSafe(scriptPath, scriptContent); hashes[scriptPath] = sha256(scriptContent); } } } // ────────────────────────────────────── // PUSH // ────────────────────────────────────── async push(customer: CustomerConfig, changes?: ChangeItem[]): Promise { const result: PushResult = { created: 0, updated: 0, deleted: 0, errors: [] }; if (!changes) { changes = await this.getChanges(customer); } if (changes.length === 0) { return result; } const client = await this.apiClientFactory(customer, false); const existingHashes = await loadHashes(customer.idn); const newHashes = { ...existingHashes }; // Load project map const mapFile = mapPath(customer.idn); if (!(await fs.pathExists(mapFile))) { result.errors.push('No project map found. Run pull first.'); return result; } const mapData = await fs.readJson(mapFile) as ProjectMap; const metadataSync = await this.syncV2FlowYamlDefinitions(client, customer, mapData, newHashes); result.created += metadataSync.created; result.updated += metadataSync.updated; result.errors.push(...metadataSync.errors); for (const change of changes) { try { if (metadataSync.syncedPaths.has(change.path)) { continue; } if (change.operation === 'modified') { // V2 flow YAML: newo_customers/{cust}/{proj}/agents/{agent}/flows/{flow}/{flow}.yaml // The flow YAML carries title, events, and state_fields inline, so // changes there must sync to the platform like V1 metadata.yaml. if (this.isV2FlowYamlPath(change.path)) { const count = await this.pushV2FlowYamlUpdate(client, change, mapData, newHashes); result.updated += count; continue; } // Detect if this is a library skill or flow skill by path const isLibrary = change.path.includes('/libraries/'); const count = isLibrary ? await this.pushV2LibrarySkillUpdate(client, change, mapData, newHashes) : await this.pushV2SkillUpdate(client, change, mapData, newHashes, customer.idn); result.updated += count; } } catch (error) { result.errors.push( `Failed to push ${change.path}: ${error instanceof Error ? error.message : String(error)}` ); } } if (metadataSync.created > 0 || metadataSync.updated > 0) { await writeFileSafe(mapFile, JSON.stringify(mapData, null, 2)); } await saveHashes(newHashes, customer.idn); if (result.created > 0 || result.updated > 0) { await this.publishAllFlows(client, mapData); } return result; } /** * Recognize the V2 flow YAML location: .../agents/{agent}/flows/{flow}/{flow}.yaml * Distinguishes it from skill scripts, library YAMLs, and attribute files. */ private isV2FlowYamlPath(p: string): boolean { const parts = p.split('/'); const file = parts[parts.length - 1]; if (!file || !file.endsWith('.yaml')) return false; // .../agents/{agent}/flows/{flow}/{flow}.yaml → last 5 parts: // agents, {agent}, flows, {flow}, {flow}.yaml if (parts.length < 5) return false; const flowsKeyword = parts[parts.length - 3]; const agentsKeyword = parts[parts.length - 5]; const flowFolder = parts[parts.length - 2] || ''; const stem = file.slice(0, -'.yaml'.length); return flowsKeyword === 'flows' && agentsKeyword === 'agents' && stem === flowFolder; } /** * Push V2 flow YAML changes. Closes GH issue #3 for newo_v2 layout. * Parses the V2 YAML, converts to V1-shaped FlowMetadata, and reuses the * shared syncFlowMetadata routine that calls PATCH/POST/DELETE per child. */ private async pushV2FlowYamlUpdate( client: AxiosInstance, change: ChangeItem, mapData: ProjectMap, newHashes: HashStore ): Promise { const parts = change.path.split('/'); const flowIdn = parts[parts.length - 2] || ''; const agentIdn = parts[parts.length - 4] || ''; const projectIdn = parts[parts.length - 6] || ''; const projectData = mapData.projects[projectIdn]; const agentData = projectData?.agents[agentIdn]; const flowData = agentData?.flows[flowIdn]; if (!flowData?.id) { this.logger.warn(`[newo_v2] Flow YAML change but flow not in project map: ${projectIdn}/${agentIdn}/${flowIdn}`); return 0; } const v2Flow = await parseV2FlowYaml(change.path); // Convert V2 → V1-shaped FlowMetadata for the shared sync routine. // V2 events lack `id` and `description`; we fill defaults so the shape // matches FlowEvent[] / FlowState[] expected by syncFlowMetadata. // Optional fields are omitted (not set to undefined) to satisfy // exactOptionalPropertyTypes. const localMeta: FlowMetadata = { id: flowData.id, idn: v2Flow.idn, title: v2Flow.title, description: v2Flow.description ?? '', default_runner_type: (v2Flow.default_runner_type as 'guidance' | 'nsl') || 'guidance', default_model: { provider_idn: v2Flow.default_provider_idn, model_idn: v2Flow.default_model_idn, }, events: (v2Flow.events || []).map((e: V2FlowEvent) => { const out: FlowEvent = { id: '', idn: e.idn, description: '', skill_selector: e.skill_selector as 'first' | 'last' | 'random' | 'all', interrupt_mode: (e.interrupt_mode || 'queue') as 'allow' | 'deny' | 'queue', ...(e.skill_idn != null ? { skill_idn: e.skill_idn } : {}), ...(e.state_idn != null ? { state_idn: e.state_idn } : {}), ...(e.integration_idn != null ? { integration_idn: e.integration_idn } : {}), ...(e.connector_idn != null ? { connector_idn: e.connector_idn } : {}), }; return out; }), state_fields: (v2Flow.state_fields || []).map((s: V2StateField) => { const out: FlowState = { id: '', idn: s.idn, title: s.title || s.idn, scope: (s.scope || 'flow') as 'flow' | 'agent' | 'project' | 'global', ...(s.default_value != null ? { default_value: s.default_value } : {}), }; return out; }), }; let remoteFlow = null; try { remoteFlow = await getFlow(client, flowData.id); } catch (error: any) { this.logger.verbose(`[newo_v2] Could not GET flow ${flowIdn}: ${error.response?.status ?? error.message}`); } const counts = emptyFlowSyncCounts(); await syncFlowMetadata(client, flowData.id, localMeta, remoteFlow, false, counts); const total = totalFlowSyncOps(counts); if (total > 0) { this.logger.info(`[newo_v2] ↑ Flow ${flowIdn}: ${describeFlowSyncCounts(counts)}`); } for (const err of counts.errors) { this.logger.warn(err); } const content = await fs.readFile(change.path, 'utf8'); newHashes[change.path] = sha256(content); return total; } /** * Reconcile inline skill definitions from V2 flow YAML before pushing scripts. * * V2 keeps skill metadata (model, runner_type, parameters) in the flow YAML, * not in a separate skill metadata file. The map only contains the remote IDs * from a previous pull, so new local skills must be created before their * callers can be published. */ private async syncV2FlowYamlDefinitions( client: AxiosInstance, customer: CustomerConfig, mapData: ProjectMap, newHashes: HashStore ): Promise<{ created: number; updated: number; syncedPaths: Set; errors: string[] }> { let created = 0; let updated = 0; const syncedPaths = new Set(); const errors: string[] = []; for (const [projectIdn, projectData] of Object.entries(mapData.projects)) { for (const [agentIdn, agentData] of Object.entries(projectData.agents)) { for (const [flowIdn, flowData] of Object.entries(agentData.flows)) { const flowYamlPath = v2FlowYamlPath(customer.idn, projectIdn, agentIdn, flowIdn); if (!(await fs.pathExists(flowYamlPath))) { continue; } let flowDef; try { flowDef = await parseV2FlowYaml(flowYamlPath); } catch (error) { this.logger.warn( `[newo_v2] Failed to parse flow YAML ${flowYamlPath}: ${error instanceof Error ? error.message : String(error)}` ); continue; } for (const skill of flowDef.skills || []) { const skillLocator = `${projectIdn}/${agentIdn}/${flowIdn}/${skill.idn}`; // Per-skill failure isolation: one broken skill must not abort the // push of every other project/flow in the workspace. try { const runnerType = this.normalizeRunnerType(skill.runner_type); const scriptPath = await this.resolveV2FlowSkillScriptPath( customer.idn, projectIdn, agentIdn, flowIdn, skill.idn, runnerType, skill.prompt_script ); if (!(await fs.pathExists(scriptPath))) { errors.push( `[newo_v2] Missing script for skill ${skillLocator}: ${scriptPath}` ); continue; } const content = await fs.readFile(scriptPath, 'utf8'); const localMetadata = this.buildV2SkillMetadataFromYaml(skill, flowDef, runnerType, flowData.skills[skill.idn]); const existingSkill = flowData.skills[skill.idn]; if (!existingSkill) { this.assertSkillModelResolved(localMetadata, skillLocator); try { const createdSkill = await createSkill(client, flowData.id, { idn: localMetadata.idn, title: localMetadata.title, prompt_script: content, runner_type: localMetadata.runner_type, model: localMetadata.model, parameters: localMetadata.parameters, path: localMetadata.path || '' }); // The create endpoint ignores inline `parameters` (verified // against the live platform) — create them explicitly. await this.createMissingSkillParameters( client, { ...localMetadata, id: createdSkill.id, parameters: [] }, localMetadata ); flowData.skills[skill.idn] = { ...localMetadata, id: createdSkill.id }; newHashes[scriptPath] = sha256(content); syncedPaths.add(scriptPath); created++; this.logger.info(`[newo_v2] Created skill: ${flowIdn}/${skill.idn}`); } catch (error) { if (!this.isAlreadyExistsApiError(error)) { throw error; } const remoteSkills = await listFlowSkills(client, flowData.id); const remoteSkill = remoteSkills.find(s => s.idn === skill.idn); if (!remoteSkill) { throw error; } const remoteMetadata: SkillMetadata = { id: remoteSkill.id, idn: remoteSkill.idn, title: remoteSkill.title, runner_type: remoteSkill.runner_type, model: remoteSkill.model, parameters: this.normalizeParameters(remoteSkill.parameters), path: remoteSkill.path }; await this.createMissingSkillParameters(client, remoteMetadata, localMetadata); await updateSkill(client, { id: remoteSkill.id, title: localMetadata.title, idn: localMetadata.idn, prompt_script: content, runner_type: localMetadata.runner_type, model: localMetadata.model, parameters: localMetadata.parameters, path: remoteSkill.path || localMetadata.path }); flowData.skills[skill.idn] = { ...localMetadata, id: remoteSkill.id, path: remoteSkill.path || localMetadata.path }; newHashes[scriptPath] = sha256(content); syncedPaths.add(scriptPath); updated++; this.logger.info(`[newo_v2] Reused existing skill: ${flowIdn}/${skill.idn}`); } continue; } const createdParameters = await this.createMissingSkillParameters(client, existingSkill, localMetadata); if (createdParameters > 0 || this.skillMetadataDiffers(existingSkill, localMetadata)) { this.assertSkillModelResolved(localMetadata, skillLocator); await updateSkill(client, { id: existingSkill.id, title: localMetadata.title, idn: localMetadata.idn, prompt_script: content, runner_type: localMetadata.runner_type, model: localMetadata.model, parameters: localMetadata.parameters, path: localMetadata.path }); flowData.skills[skill.idn] = { ...localMetadata, id: existingSkill.id }; newHashes[scriptPath] = sha256(content); syncedPaths.add(scriptPath); updated++; this.logger.info(`[newo_v2] Updated skill metadata: ${flowIdn}/${skill.idn}`); } } catch (error) { errors.push( `Failed to sync skill ${skillLocator}: ${error instanceof Error ? error.message : String(error)}` ); } } } } } return { created, updated, syncedPaths, errors }; } private async createMissingSkillParameters( client: AxiosInstance, existing: SkillMetadata, local: SkillMetadata ): Promise { const existingNames = new Set(this.normalizeParameters(existing.parameters).map(p => p.name)); let created = 0; for (const parameter of local.parameters) { if (existingNames.has(parameter.name)) { continue; } try { await createSkillParameter(client, existing.id, { name: parameter.name, default_value: parameter.default_value ?? '' }); created++; this.logger.info(`[newo_v2] Created skill parameter: ${local.idn}/${parameter.name}`); } catch (error) { if (!this.isAlreadyExistsApiError(error)) { throw error; } } existingNames.add(parameter.name); } return created; } /** * Detect "resource already exists" API errors. * * Matches only on the precise phrases the platform actually returns * ("already exists", "duplicate key"). Loose substrings like "exist" * would otherwise sweep up unrelated "does not exist" / "doesn't exist" * errors and trigger an incorrect reuse fallback. */ private isAlreadyExistsApiError(error: unknown): boolean { const response = (error as { response?: { status?: number; data?: unknown } } | null | undefined)?.response; const status = response?.status; if (status !== 400 && status !== 409 && status !== 422) { return false; } const haystack = JSON.stringify( response?.data ?? (error instanceof Error ? error.message : String(error)) ).toLowerCase(); return haystack.includes('already exists') || haystack.includes('duplicate key'); } private normalizeRunnerType(runnerType: string | undefined): RunnerType { return runnerType === 'nsl' ? 'nsl' : 'guidance'; } private normalizeParameters(parameters: readonly SkillParameter[] | undefined): SkillParameter[] { return (parameters || []).map(p => ({ name: p.name, default_value: p.default_value ?? '' })); } /** * Fail fast if no model could be resolved for a V2 skill. * * `buildV2SkillMetadataFromYaml` falls back to empty strings when neither * the skill nor the flow declare a model. The platform rejects empty * model_idn/provider_idn at creation/update time, but the error it returns * is generic — we surface a clearer message before issuing the request. */ private assertSkillModelResolved(metadata: SkillMetadata, locator: string): void { if (!metadata.model.model_idn || !metadata.model.provider_idn) { throw new Error( `[newo_v2] Cannot resolve model for skill ${locator}: ` + `model_idn="${metadata.model.model_idn}", provider_idn="${metadata.model.provider_idn}". ` + `Set either skill.model.* or flow default_model_idn/default_provider_idn in the flow YAML.` ); } } private buildV2SkillMetadataFromYaml( skill: V2InlineSkill, flowDef: Awaited>, runnerType: RunnerType, existing?: SkillMetadata ): SkillMetadata { return { id: existing?.id || '', idn: skill.idn, title: skill.title || '', runner_type: runnerType, model: { model_idn: skill.model?.model_idn || flowDef.default_model_idn || '', provider_idn: skill.model?.provider_idn || flowDef.default_provider_idn || '' }, parameters: this.normalizeParameters(skill.parameters), path: existing?.path || '' }; } private skillMetadataDiffers(existing: SkillMetadata, local: SkillMetadata): boolean { // Compare model/parameters field-by-field, never via JSON.stringify of the // raw objects: the map stores model keys in platform API order // (provider_idn first) while YAML-built metadata uses model_idn first, and // a key-order-sensitive comparison flags every skill as changed. const paramsKey = (params: readonly SkillParameter[] | undefined): string => JSON.stringify( this.normalizeParameters(params).sort((a, b) => a.name.localeCompare(b.name)) ); return ( existing.title !== local.title || existing.runner_type !== local.runner_type || existing.model.model_idn !== local.model.model_idn || existing.model.provider_idn !== local.model.provider_idn || paramsKey(existing.parameters) !== paramsKey(local.parameters) ); } private async resolveV2FlowSkillScriptPath( customerIdn: string, projectIdn: string, agentIdn: string, flowIdn: string, skillIdn: string, runnerType: RunnerType, promptScript?: string ): Promise { if (promptScript) { const fromPromptScript = `${v2AgentDir(customerIdn, projectIdn, agentIdn)}/${promptScript}`; if (await fs.pathExists(fromPromptScript)) { return fromPromptScript; } } return v2SkillScriptPath(customerIdn, projectIdn, agentIdn, flowIdn, skillIdn, runnerType); } /** * Push a V2 skill update * * V2 path: newo_customers/{cust}/{proj}/agents/{agent}/flows/{flow}/skills/{skill}.nsl */ private async pushV2SkillUpdate( client: AxiosInstance, change: ChangeItem, mapData: ProjectMap, newHashes: HashStore, customerIdn: string ): Promise { const target = await this.resolveV2SkillTargetForScriptPath(customerIdn, change.path, mapData) || this.resolveV2SkillTargetFromCanonicalPath(change.path, mapData); const skillData = target?.skillData; if (!target || !skillData) { throw new Error(`Skill not found in project map (path: ${change.path})`); } // Read updated script content const content = await fs.readFile(change.path, 'utf8'); // Update via V1 API await updateSkill(client, { id: skillData.id, title: skillData.title, idn: skillData.idn, prompt_script: content, runner_type: skillData.runner_type, model: skillData.model, parameters: skillData.parameters, path: skillData.path }); newHashes[change.path] = sha256(content); this.logger.info(`[newo_v2] Pushed: ${target.skillIdn}`); return 1; } private normalizePathForComparison(filePath: string): string { return path.resolve(filePath).replace(/\\/g, '/'); } private async resolveV2SkillTargetForScriptPath( customerIdn: string, scriptPath: string, mapData: ProjectMap ): Promise { const normalizedScriptPath = this.normalizePathForComparison(scriptPath); for (const [projectIdn, projectData] of Object.entries(mapData.projects)) { for (const [agentIdn, agentData] of Object.entries(projectData.agents)) { for (const [flowIdn, flowData] of Object.entries(agentData.flows)) { const flowYamlPath = v2FlowYamlPath(customerIdn, projectIdn, agentIdn, flowIdn); if (!(await fs.pathExists(flowYamlPath))) { continue; } let flowDef; try { flowDef = await parseV2FlowYaml(flowYamlPath); } catch { continue; } for (const skill of flowDef.skills || []) { const runnerType = this.normalizeRunnerType(skill.runner_type || flowData.skills[skill.idn]?.runner_type); const resolvedScriptPath = await this.resolveV2FlowSkillScriptPath( customerIdn, projectIdn, agentIdn, flowIdn, skill.idn, runnerType, skill.prompt_script ); if (this.normalizePathForComparison(resolvedScriptPath) === normalizedScriptPath) { return { projectIdn, agentIdn, flowIdn, skillIdn: skill.idn, skillData: flowData.skills[skill.idn] }; } } } } } return null; } private resolveV2SkillTargetFromCanonicalPath( scriptPath: string, mapData: ProjectMap ): V2FlowSkillTarget | null { // Parse canonical V2 path: // .../newo_customers/{cust}/{proj}/agents/{agent}/flows/{flow}/skills/{skillFile} const pathParts = scriptPath.split('/'); const skillFileName = pathParts[pathParts.length - 1] || ''; const skillIdn = skillFileName.replace(/\.(nsl|nslg|jinja|guidance)$/, ''); // skills/ -> flow/ -> flows/ -> agent/ -> agents/ -> project/ const flowIdn = pathParts[pathParts.length - 3] || ''; const agentIdn = pathParts[pathParts.length - 5] || ''; const projectIdn = pathParts[pathParts.length - 7] || ''; const projectData = mapData.projects[projectIdn]; const agentData = projectData?.agents[agentIdn]; const flowData = agentData?.flows[flowIdn]; return { projectIdn, agentIdn, flowIdn, skillIdn, skillData: flowData?.skills[skillIdn] }; } /** * Push a V2 library skill update * Path: .../newo_customers/{cust}/{proj}/libraries/{lib}/skills/{skillFile} */ private async pushV2LibrarySkillUpdate( client: AxiosInstance, change: ChangeItem, mapData: ProjectMap, newHashes: HashStore ): Promise { const pathParts = change.path.split('/'); const skillFileName = pathParts[pathParts.length - 1] || ''; const skillIdn = skillFileName.replace(/\.(nsl|nslg|jinja|guidance)$/, ''); // skills/ -> lib/ -> libraries/ -> project/ const libIdn = pathParts[pathParts.length - 3] || ''; const projectIdn = pathParts[pathParts.length - 5] || ''; const projectData = mapData.projects[projectIdn]; const libData = projectData?.libraries?.[libIdn]; const skillData = libData?.skills[skillIdn]; if (!skillData || !libData) { throw new Error(`Library skill ${skillIdn} not found in project map (path: ${change.path})`); } const content = await fs.readFile(change.path, 'utf8'); await updateLibrarySkill(client, libData.id, skillData.id, { prompt_script: content, }); newHashes[change.path] = sha256(content); this.logger.info(`[newo_v2] Pushed library skill: ${libIdn}/${skillIdn}`); return 1; } /** * Publish all flows */ private async publishAllFlows(client: AxiosInstance, mapData: ProjectMap): Promise { for (const projectData of Object.values(mapData.projects)) { for (const agentData of Object.values(projectData.agents)) { for (const [flowIdn, flowData] of Object.entries(agentData.flows)) { if (flowData.id) { try { await publishFlow(client, flowData.id, { version: '1.0', description: 'Published via NEWO CLI (newo_v2)', type: 'public' }); this.logger.verbose(`[newo_v2] Published flow: ${flowIdn}`); } catch { this.logger.warn(`[newo_v2] Failed to publish flow ${flowIdn}`); } } } } } } // ────────────────────────────────────── // STATUS / CHANGES // ────────────────────────────────────── async getChanges(customer: CustomerConfig): Promise[]> { const changes: ChangeItem[] = []; const mapFile = mapPath(customer.idn); if (!(await fs.pathExists(mapFile))) { return changes; } const hashes = await loadHashes(customer.idn); const mapData = await fs.readJson(mapFile) as ProjectMap; // Scan V2 directory structure for changed skill scripts for (const [projectIdn, projectData] of Object.entries(mapData.projects)) { // Flow skills for (const [agentIdn, agentData] of Object.entries(projectData.agents)) { for (const [flowIdn, _flowData] of Object.entries(agentData.flows)) { // V2 stores flow events / state_fields / title inline in the flow // YAML. Detect changes here so push() can sync them (GH issue #3). const flowYamlPath = v2FlowYamlPath(customer.idn, projectIdn, agentIdn, flowIdn); if (await fs.pathExists(flowYamlPath)) { const content = await fs.readFile(flowYamlPath, 'utf8'); const currentHash = sha256(content); const storedHash = hashes[flowYamlPath]; if (storedHash !== currentHash) { changes.push({ item: {} as LocalProjectData, operation: 'modified', path: flowYamlPath }); } } } for (const [flowIdn, flowData] of Object.entries(agentData.flows)) { const flowYamlSkills = await this.loadLocalV2FlowSkills(customer.idn, projectIdn, agentIdn, flowIdn); const skillIdns = new Set([ ...Object.keys(flowData.skills), ...flowYamlSkills.keys() ]); for (const skillIdn of skillIdns) { const yamlSkill = flowYamlSkills.get(skillIdn); const skillMeta = flowData.skills[skillIdn]; const runnerType = this.normalizeRunnerType(yamlSkill?.runner_type || skillMeta?.runner_type); const scriptPath = yamlSkill ? await this.resolveV2FlowSkillScriptPath( customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, runnerType, yamlSkill.prompt_script ) : v2SkillScriptPath( customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, runnerType ); if (await fs.pathExists(scriptPath)) { const content = await fs.readFile(scriptPath, 'utf8'); const currentHash = sha256(content); const storedHash = hashes[scriptPath]; if (storedHash !== currentHash) { changes.push({ item: {} as LocalProjectData, operation: 'modified', path: scriptPath }); } } } } } // Library skills if (projectData.libraries) { for (const [libIdn, libData] of Object.entries(projectData.libraries)) { for (const [skillIdn, skillMeta] of Object.entries(libData.skills)) { const scriptPath = v2LibrarySkillScriptPath( customer.idn, projectIdn, libIdn, skillIdn, skillMeta.runner_type ); if (await fs.pathExists(scriptPath)) { const content = await fs.readFile(scriptPath, 'utf8'); const currentHash = sha256(content); const storedHash = hashes[scriptPath]; if (storedHash !== currentHash) { changes.push({ item: {} as LocalProjectData, operation: 'modified', path: scriptPath }); } } } } } } return changes; } private async loadLocalV2FlowSkills( customerIdn: string, projectIdn: string, agentIdn: string, flowIdn: string ): Promise> { const flowYamlPath = v2FlowYamlPath(customerIdn, projectIdn, agentIdn, flowIdn); if (!(await fs.pathExists(flowYamlPath))) { return new Map(); } try { const flowDef = await parseV2FlowYaml(flowYamlPath); return new Map((flowDef.skills || []).map(skill => [skill.idn, skill])); } catch { return new Map(); } } async validate(customer: CustomerConfig, _items: LocalProjectData[]): Promise { const errors: ValidationError[] = []; const mapFile = mapPath(customer.idn); if (!(await fs.pathExists(mapFile))) { errors.push({ field: 'projectMap', message: 'No project map found. Run pull first.' }); return { valid: false, errors }; } const mapData = await fs.readJson(mapFile) as ProjectMap; // Validate V2 skill files exist for (const [projectIdn, projectData] of Object.entries(mapData.projects)) { for (const [agentIdn, agentData] of Object.entries(projectData.agents)) { for (const [flowIdn, flowData] of Object.entries(agentData.flows)) { const flowYamlPath = v2FlowYamlPath(customer.idn, projectIdn, agentIdn, flowIdn); let localYamlSkills: Map | undefined; if (await fs.pathExists(flowYamlPath)) { try { const flowDef = await parseV2FlowYaml(flowYamlPath); localYamlSkills = new Map((flowDef.skills || []).map(s => [s.idn, s])); const skillIdns = new Set([ ...Object.keys(flowData.skills), ...localYamlSkills.keys() ]); for (const skillIdn of skillIdns) { const localYamlSkill = localYamlSkills.get(skillIdn); const skillMeta = flowData.skills[skillIdn]; if (!localYamlSkill) { errors.push({ field: `skill.${skillIdn}`, message: `Skill exists in project map but is missing from flow YAML: ${flowYamlPath}`, path: flowYamlPath }); continue; } const runnerType = this.normalizeRunnerType( localYamlSkill.runner_type || skillMeta?.runner_type ); const scriptPath = await this.resolveV2FlowSkillScriptPath( customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, runnerType, localYamlSkill.prompt_script ); if (!(await fs.pathExists(scriptPath))) { errors.push({ field: `skill.${localYamlSkill.idn}`, message: `Script file not found: ${scriptPath}`, path: scriptPath }); } } } catch { localYamlSkills = undefined; } } for (const [skillIdn, skillMeta] of Object.entries(flowData.skills)) { if (localYamlSkills) { continue; } const scriptPath = v2SkillScriptPath( customer.idn, projectIdn, agentIdn, flowIdn, skillIdn, skillMeta.runner_type ); if (!(await fs.pathExists(scriptPath))) { errors.push({ field: `skill.${skillIdn}`, message: `Script file not found: ${scriptPath}`, path: scriptPath }); } } } } } return { valid: errors.length === 0, errors }; } async getStatus(customer: CustomerConfig): Promise { const changes = await this.getChanges(customer); return { resourceType: this.resourceType, displayName: this.displayName, changedCount: changes.length, changes: changes.map(c => ({ path: c.path, operation: c.operation })) }; } } // ── V2 Attributes Formatting ── /** * Map V1 API value_type to V2 export format * V1 API: "string", "bool", "AttributeValueTypes.string", etc. * V2 export: "ValueType.STRING", "ValueType.BOOL", etc. */ function toV2ValueType(apiValueType: string): string { // Already in V2 format if (apiValueType.startsWith('ValueType.')) return apiValueType; // Strip AttributeValueTypes. prefix if present const raw = apiValueType.replace(/^AttributeValueTypes\./, ''); const mapping: Record = { 'string': 'ValueType.STRING', 'bool': 'ValueType.BOOL', 'number': 'ValueType.NUMBER', 'enum': 'ValueType.ENUM', 'json': 'ValueType.JSON', }; return mapping[raw.toLowerCase()] || `ValueType.${raw.toUpperCase()}`; } /** * Format attributes as V2 YAML with: * - Sorted by idn alphabetically * - value_type as !enum "ValueType.X" * - Proper quoting */ function formatV2AttributesYaml(attrs: Array<{ idn: string; value: any; title?: string | undefined; description?: string | undefined; group?: string | undefined; is_hidden?: boolean | undefined; possible_values?: any[] | undefined; value_type?: string | undefined; }>): string { // Sort alphabetically by idn const sorted = [...attrs].sort((a, b) => a.idn.localeCompare(b.idn)); const cleaned = sorted.map(a => ({ idn: a.idn, value: a.value, title: a.title || '', description: a.description || '', group: a.group || '', is_hidden: a.is_hidden ?? false, possible_values: a.possible_values || [], value_type: new V2EnumValue(toV2ValueType(a.value_type || 'string')), })); const enumType = new yaml.Type('!enum', { kind: 'scalar', instanceOf: V2EnumValue, resolve: () => true, construct: (data: string) => new V2EnumValue(data), represent: (data: unknown) => data instanceof V2EnumValue ? data.value : String(data), }); const schema = yaml.DEFAULT_SCHEMA.extend([enumType]); // Use lineWidth: -1 to prevent folding multiline strings (preserve |- literal block style) const rawYaml = yaml.dump({ attributes: cleaned }, { indent: 2, quotingType: '"', forceQuotes: false, lineWidth: -1, noRefs: true, sortKeys: false, schema, }); // Fix !enum quoting: js-yaml outputs `!enum ValueType.STRING` but V2 ZIP uses `!enum "ValueType.STRING"` const enumFixed = rawYaml.replace(/!enum (\S+)/g, '!enum "$1"'); // Patch long-line wrapping to match pyyaml style return patchYamlToPyyaml(enumFixed); } /** Wrapper class for !enum YAML tag */ class V2EnumValue { constructor(public value: string) {} } /** * Factory function for creating V2ProjectSyncStrategy */ export function createV2ProjectSyncStrategy( apiClientFactory: ApiClientFactory, logger: ILogger ): V2ProjectSyncStrategy { return new V2ProjectSyncStrategy(apiClientFactory, logger); }