import * as vscode from 'vscode'; import { v4 as uuidv4 } from 'uuid'; import * as projectmanager from './ProjectManager'; import * as path from 'path'; /** * @interface FsProjectManager * * @description Filesystem project manager class let handling of an Edge.AI project data model stored within local fileystem */ export class FsProjectManager extends projectmanager.IProjectManager { protected __project : any; public set _project(value: any) { this.__project = value; this._parsedProject = projectmanager.Project.deserialize(this.__project, this.uri); } public get _project(): any { return this.__project; } protected _parsedProject: projectmanager.Project | undefined; public get project(): projectmanager.Project { if(!this._parsedProject) throw new Error('No project set'); return this._parsedProject; } protected _onDidChangeProject: vscode.EventEmitter = new vscode.EventEmitter(); public onDidChangeProject : vscode.Event; private projectFileWatcher : vscode.FileSystemWatcher | undefined; constructor(public readonly baseUri: vscode.Uri, readonly projectSchemaUri : vscode.Uri) { super(baseUri); this.onDidChangeProject = this._onDidChangeProject.event; } /** * Inerithed methods */ /** * @name connect * @description Connect to project json file by instantianting filesystem listeners, and set it as current project * * @param uri Filesystem URI referring to project json file */ public async connect(uri : vscode.Uri) : Promise { this.uri = uri; const pattern = "**" + uri.fsPath.replace(this.baseUri.fsPath, "").replace(new RegExp(/\\/, 'g'), `/`); this.projectFileWatcher = vscode.workspace.createFileSystemWatcher(pattern, true, false, true); this.projectFileWatcher.onDidChange(async ()=> { await this.load(); }); } /** * @name load * @description Load project json file from filesystem and fire project change event */ public async load() : Promise { let buf : Uint8Array = await vscode.workspace.fs.readFile(this.uri); let str = Buffer.from(buf).toString(); this._project = await JSON.parse(str); this._onDidChangeProject.fire(this.project); } /** * @name newProject * @description Load project json file from filesystem and fire project change event * * @param projectId Project ID (optional). If not provided, it will be generated following UUID v4 format * @param name Project name * @param description Project description * @param owner Project owner ID (not required). If provided, it will be ignored * @param version Project version (optional). If not provided, will be assigned the default version (i.e. "0.0.1") */ public async newProject(projectId: string| undefined = undefined, name: string, description: string, owner: string | undefined, version: string | undefined = undefined) { this._project = { $schema: path.join(this.projectSchemaUri.fsPath), ai_project_name : name, uuid: projectId || uuidv4(), description : description, version: version || "0.0.1", inputs: [], models: [], outputs: [], deployments: [] }; return this.project; } /** * @name addModel * @description Create a new model for the current project * * @param modelId Model ID (optional). If not provided, it will be generated following UUID v4 format * @param inputId Input ID (optional). If not provided, it will be generated following UUID v4 format * @param outputId Output ID (optional). If not provided, it will be generated following UUID v4 format * @param name Model name * @param stage Model stage * @param inputType Model input type * @param modelType Model type * @param outputType Model output type * @param trainingType Model training type * @param modelMetadata Model metadata * @param inputTools Model input tools * @param outputTools Model output tools * @returns Descriptor for the model (just) created */ public addModel(modelId: string | undefined, inputId: string | undefined, outputId: string | undefined, name: string, stage: projectmanager.ModelStage, inputType: projectmanager.InputType, modelType: projectmanager.ModelType, outputType: projectmanager.OutputType, trainingType: projectmanager.TrainingType, modelMetadata: any, inputTools: projectmanager.Tool[], outputTools: projectmanager.Tool[]): projectmanager.Model{ if(this._project.models.find((m:any) => m.name == name)) throw new Error("Model named '"+name+"' already exist"); modelId = modelId || uuidv4(); const rawInput : any = { "uuid": inputId || uuidv4(), "input_type": inputType.toString(), "model_ref": modelId, "logs": [], "augmentation": [], "tools": inputTools }; this._project.inputs.push(rawInput); const rawModel : any = { "uuid": modelId, "name": name, "stage": stage.toString(), "model_type": modelType.toString(), "training_type": trainingType.toString(), "retraining": false, "model_metadata": modelMetadata, "experiments": [], "tools": [] }; this._project.models.push(rawModel); const rawOutput : any = { "uuid": outputId || uuidv4(), "output_type": outputType.toString(), "model_ref": modelId, "tests": [], "best_test": null, "tools": outputTools, "events":[], "ui": null }; this._project.outputs.push(rawOutput); const input : projectmanager.Input = projectmanager.Input.deserialize(rawInput); const output: projectmanager.Output = projectmanager.Output.deserialize(rawOutput); const model: projectmanager.Model = projectmanager.Model.deserialize(rawModel,input, output, []); return model; } /** * @name updateModel * @description Update existing model with input parameters (if provided) * * @param modelId Model ID * @param description Model description (optional). If not provided, field will be not updated * @param stage Model stage (optional). If not provided, field will be not updated * @param classes Model classes (optional). If not provided, field will be not updated * @param augmentations Model augmentations (optional). If not provided, field will be not updated */ public updateModel(modelId: string, description: string | undefined, stage: projectmanager.ModelStage | undefined, classes: string[] | undefined, augmentations: string[] | undefined = undefined): void { let model : any = this._project.models.find( (model: any) => model.uuid == modelId ); if(stage){ model.stage = stage.toString(); } if(classes){ model.model_metadata.classes = classes; } if(augmentations){ let input : any = this._project.inputs.find( (input: any) => input.model_ref == modelId ); input.augmentation = augmentations; } } /** * @name removeModel * @description Remove existing model * * @param modelId Model ID */ public removeModel(modelId: string) : void { this._project.inputs = this._project.inputs.filter( (input: any) => input.model_ref != modelId ); this._project.models = this._project.models.filter( (model: any) => model.uuid != modelId ); this._project.outputs = this._project.outputs.filter( (output: any) => output.model_ref != modelId ); } /** * @name addExperiment * @description Create a new experiment for a given model * * @param modelId Model ID * @param experimentId Experiment ID (optional). If not provided, it will be generated following UUID v4 format * @param name Exeperiment name * @param description Experiment description * @param modelDevFile Model development filename * @returns Descriptor for the experiment (just) created */ public addExperiment(modelId: string, experimentId: string | undefined, name: string, description: string, modelDevFile: string | null): projectmanager.Experiment { let model = this._project.models.find((m:any) => m.uuid == modelId); if(!model) throw new Error("Model '"+modelId+"' doesn't exist"); if(model.experiments.find((e:any) => e.name == name)) throw new Error("Experiment named '"+name+"' already exist"); experimentId = experimentId || uuidv4(); const rawExperiment = { "uuid": experimentId, "name": name, "description": description, "model_dev_file": modelDevFile }; model.experiments.push({ "uuid": experimentId, "name": name, "description": description, "model_dev_file": modelDevFile }); model.stage = projectmanager.ModelStage.modeling.toString(); return projectmanager.Experiment.deserialize(rawExperiment, []); } /** * @name updateExperiment * @description Update existing experiment with input parameters (if provided) * * @param modelId Model ID * @param experimentId Experiment ID * @param description Experiment description (optional). If not provided, field will be not updated * @param classes Model development file name (optional). If not provided, field will be not updated */ public updateExperiment(modelId: string, experimentId: string, description: string | undefined, modelDevFile: string | null | undefined){ let model = this._project.models.find((m:any) => m.uuid == modelId); if(!model) throw new Error("Model '"+modelId+"' doesn't exist"); let experiment = model.experiments.find((e:any) => e.uuid == experimentId); if(!experiment){ throw new Error("Experiment '"+experimentId+"' doesn't exist"); } experiment.description = description || experiment.description; experiment.model_dev_file = (modelDevFile || modelDevFile==null) ? modelDevFile : experiment.model_dev_file } /** * @name removeExperiment * @description Remove existing experiment * * @param modelId Model ID * @param experimentId Experiment ID */ public removeExperiment(modelId: string, experimentId: string) : void { let model : any = this._project.models.find( (model: any) => model.uuid == modelId ); let output : any = this._project.outputs.find( (output: any) => output.model_ref == modelId ); model.experiments = model.experiments.filter( (experiment: any) => experiment.uuid != experimentId ); output.tests = output.tests.filter( (test: any) => test.experiment_ref != experimentId ); } /** * @name addTest * @description Create a new test for a given experiment * * @param modelId Model ID * @param experimentId Experiment ID * @param testId Test ID (optional). If not provided, it will be generated following UUID v4 format * @param name Test name * @param modelFile Model file name * @returns Descriptor for the test (just) created */ public addTest(modelId: string, experimentId: string, testId: string | undefined, name: string, modelFile: string | null): projectmanager.Test { const output : any = this._project.outputs.find( (o: any) => o.model_ref == modelId ); const test : any | undefined = output.tests.find( (t: any) => t.name == name && t.experiment_ref == experimentId); if(test) throw new Error("Test named '"+name+"' already exist"); const rawTest : any = { "uuid": testId || uuidv4(), "name": name, "experiment_ref": experimentId, "model_file": modelFile, "parameters": {}, "reports": [], "outputs": [] }; output.tests.push(rawTest); return projectmanager.Test.deserialize(rawTest, false); } /** * @name updateTest * @description Update existing test with input parameters (if provided) * * @param modelId Model ID * @param testId Test ID * @param parameters Parameters used to generate reports and/or output. If not provided, field will be not updated * @param reports Report filenames. If not provided, field will be not updated * @param outputs Ouputs filenames. If not provided, field will be not updated * @param isBest Is the best test? (default false) */ public updateTest(modelId: string, testId: string, parameters: any, reports: string[] | undefined, outputs: string[] | undefined, isBest: boolean | undefined = undefined): any { let output = this._project.outputs.find((o:any) => o.model_ref == modelId); if(!output){ throw new Error("Ouput referring to model '" + modelId + "' doesn't exist"); } let test = output.tests.find((t:any) => t.uuid == testId); if(!test) throw new Error("Test '"+testId+"' doesn't exist"); test.parameters = parameters || test.parameters; test.reports = reports || test.reports; test.outputs = outputs || test.outputs; if(isBest === true){ output.best_test = testId } else if(isBest === false) { if(output.best_test===testId) { output.best_test = null; } } } /** * @name removeTest * @description Remove existing test * * @param modelId Model ID * @param experimentId Experiment ID * @param testId Test ID */ public removeTest(modelId: string, experimentId: string, testId: string) : void { let output : any = this._project.outputs.find( (output: any) => output.model_ref == modelId ); output.tests = output.tests.filter( (test: any) => test.experiment_ref != experimentId || test.uuid != testId); } /** * Additional methods */ /** * @name commit * @description Commit local changes by storing them on the project file current referred */ public async commit(): Promise { let projectString = JSON.stringify(this._project, null, 3); let buf = Buffer.from(projectString); await vscode.workspace.fs.writeFile(this.uri, buf); } /** * @name disconnect * @description Disconnect from current project disposing filesystem watchers */ public async disconnect() : Promise { this.projectFileWatcher?.dispose(); this._onDidChangeProject.fire(undefined); } /** * @name setBestTest * @description Set current test as best for the model * * @param modelId Model ID * @param testId Test ID */ public setBestTest(modelId: string, testId: string) : void { let output : any = this._project.outputs.find( (output: any) => output.model_ref == modelId ); output.best_test = testId; } /** * @name updateTestReports * @description Update report filenames and related parameters for the test * * @param modelId Model ID * @param testId Test ID * @param parameters Parameters used to generate the reports * @param reports New report filenames */ public updateTestReports(modelId: string, testId: string, parameters: any, reports: Array) { let output : any = this._project.outputs.find( (output: any) => output.model_ref == modelId ); let test : any = output.tests.find( (test: any) => test.uuid == testId); test.parameters = parameters; test.reports = reports; } /** * @name updateTestOutputs * @description Update output filenames and related parameters for the test * * @param modelId Model ID * @param testId Test ID * @param parameters Parameters used to generate the outputs * @param reports New output filenames */ public updateTestOutputs(modelId: string, testId: string, parameters: any, outputs: Array) { let output : any = this._project.outputs.find( (output: any) => output.model_ref == modelId ); let test : any = output.tests.find( (test: any) => test.uuid == testId); test.parameters = parameters; test.outputs = outputs; } /** * @name addModelAugmentation * @description Add augmentation filename for the model * * @param modelId Model ID * @param name Augmentation filename */ public addModelAugmentation(modelId: string, name: string) : void { let input : any = this._project.inputs.find( (input: any) => input.model_ref == modelId ); if(!input.augmentation.find( (n:string) => n == name)) input.augmentation.push(name); } /** * @name removeModelAugmentation * @description Remove augmentation filename for the model * * @param modelId Model ID * @param name Augmentation filename */ public removeModelAugmentation(modelId: string, name: string) : void { let input : any = this._project.inputs.find( (input: any) => input.model_ref == modelId ); input.augmentation = input.augmentation.filter( (n:string) => n != name); } }