/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { action, makeObservable, observable } from 'mobx';
import type {
LegendDataCubeApplicationStore,
LegendDataCubeBaseStore,
} from '../LegendDataCubeBaseStore.js';
import {
type DataCubeAlertService,
type DataCubeAPI,
type DataCubeLayoutService,
type DataCubeTaskService,
DataCubeSpecification,
DEFAULT_ALERT_WINDOW_CONFIG,
type DisplayState,
DataCubeSpecificationOptions,
type DataCubeSource,
} from '@finos/legend-data-cube';
import { LegendDataCubeCreatorState } from './LegendDataCubeCreatorState.js';
import {
PersistentDataCube,
type LightPersistentDataCube,
type V1_EngineServerClient,
type V1_PureGraphManager,
} from '@finos/legend-graph';
import {
ActionState,
assertErrorThrown,
formatDate,
isString,
returnUndefOnError,
UnsupportedOperationError,
uuid,
} from '@finos/legend-shared';
import type { LegendDataCubeDataCubeEngine } from '../LegendDataCubeDataCubeEngine.js';
import { LegendDataCubeSaver } from '../../components/builder/LegendDataCubeSaver.js';
import {
generateBuilderRoute,
LEGEND_DATA_CUBE_ROUTE_PATTERN_TOKEN,
} from '../../__lib__/LegendDataCubeNavigation.js';
import { LegendDataCubeLoaderState } from './LegendDataCubeLoaderState.js';
import {
LegendDataCubeUserDataKey,
RECENTLY_VIEWED_DATA_CUBES_LIMIT,
} from '../../__lib__/LegendDataCubeUserData.js';
import type { DepotServerClient } from '@finos/legend-server-depot';
import { LegendDataCubeBlockingWindowState } from '../../components/LegendDataCubeBlockingWindow.js';
import { LegendDataCubeDeleteConfirmation } from '../../components/builder/LegendDataCubeDeleteConfirmation.js';
import {
LegendDataCubeAbout,
LegendDataCubeReleaseLogManager,
} from '../../components/builder/LegendDataCubeBuilder.js';
import { LegendDataCubeSourceViewer } from '../../components/builder/LegendDataCubeSourceViewer.js';
import { LOCAL_FILE_QUERY_DATA_CUBE_SOURCE_TYPE } from '../model/LocalFileDataCubeSource.js';
import type { LegendDataCubeSourceLoaderState } from './source/LegendDataCubeSourceLoaderState.js';
import { LocalFileDataCubeSourceLoaderState } from './source/LocalFileDataCubeSourceLoaderState.js';
import { LEGEND_DATACUBE_APP_EVENT } from '../../__lib__/LegendDataCubeEvent.js';
import { LegendDataCubeQueryEditor } from '../../components/builder/LegendDataCubeQueryEditor.js';
import type {
LakehouseContractServerClient,
LakehouseIngestServerClient,
LakehousePlatformServerClient,
} from '@finos/legend-server-lakehouse';
import { LegendDataCubeEditSourceQuery } from '../../components/builder/LegendDataCubeEditSourceQuery.js';
import type { LegendDataCubeCodeEditorState } from './LegendDataCubeCodeEditorState.js';
import {
LakehouseConsumerDataCubeSource,
RawLakehouseConsumerDataCubeSource,
} from '../model/LakehouseConsumerDataCubeSource.js';
import {
LakehouseProducerDataCubeSource,
RawLakehouseProducerDataCubeSource,
} from '../model/LakehouseProducerDataCubeSource.js';
export class LegendDataCubeBuilderState {
readonly uuid = uuid();
readonly startTime = Date.now();
readonly _store!: LegendDataCubeBuilderStore;
readonly initialSpecification!: DataCubeSpecification;
persistentDataCube?: PersistentDataCube | undefined;
dataCube?: DataCubeAPI | undefined;
source?: DataCubeSource | undefined;
constructor(
store: LegendDataCubeBuilderStore,
specification: DataCubeSpecification,
persistentDataCube?: PersistentDataCube | undefined,
) {
makeObservable(this, {
persistentDataCube: observable,
setPersistentDataCube: action,
dataCube: observable,
setDataCube: action,
source: observable,
setSource: action,
});
this._store = store;
this.initialSpecification = specification;
this.persistentDataCube = persistentDataCube;
}
setPersistentDataCube(val: PersistentDataCube | undefined) {
this.persistentDataCube = val;
}
setDataCube(val: DataCubeAPI | undefined) {
this.dataCube = val;
}
setSource(val: DataCubeSource | undefined) {
this.source = val;
}
}
export type LegendDataCubeSaveOptions = {
syncName?: boolean | undefined;
autoEnableCache?: boolean | undefined;
};
export class LegendDataCubeBuilderStore {
readonly application: LegendDataCubeApplicationStore;
readonly baseStore: LegendDataCubeBaseStore;
readonly engine: LegendDataCubeDataCubeEngine;
readonly depotServerClient: DepotServerClient;
readonly platformServerClient: LakehousePlatformServerClient;
readonly ingestServerClient: LakehouseIngestServerClient;
readonly contractServerClient: LakehouseContractServerClient;
readonly engineServerClient: V1_EngineServerClient;
readonly graphManager: V1_PureGraphManager;
readonly taskService: DataCubeTaskService;
readonly layoutService: DataCubeLayoutService;
readonly alertService: DataCubeAlertService;
readonly initializeState = ActionState.create();
readonly aboutDisplay: DisplayState;
readonly releaseLogDisplay: DisplayState;
readonly releaseNotesDisplay: DisplayState;
readonly creator: LegendDataCubeCreatorState;
readonly saveState = ActionState.create();
readonly saverDisplay: LegendDataCubeBlockingWindowState;
readonly codeEditorState = ActionState.create();
readonly codeEditorDisplay: LegendDataCubeBlockingWindowState;
readonly queryEditorState = ActionState.create();
readonly queryEditorDisplay: LegendDataCubeBlockingWindowState;
readonly deleteState = ActionState.create();
dataCubeToDelete?: LightPersistentDataCube | PersistentDataCube | undefined;
readonly deleteConfirmationDisplay: LegendDataCubeBlockingWindowState;
readonly loadState = ActionState.create();
readonly loader: LegendDataCubeLoaderState;
builder?: LegendDataCubeBuilderState | undefined;
readonly sourceViewerDisplay: DisplayState;
sourceLoader?: LegendDataCubeSourceLoaderState | undefined;
private passedFirstLoad = false;
constructor(baseStore: LegendDataCubeBaseStore) {
makeObservable(this, {
builder: observable,
setBuilder: action,
sourceLoader: observable,
setSourceLoader: action,
dataCubeToDelete: observable,
setDataCubeToDelete: action,
});
this.application = baseStore.application;
this.baseStore = baseStore;
this.engine = baseStore.engine;
this.depotServerClient = baseStore.depotServerClient;
this.engineServerClient = baseStore.engineServerClient;
this.platformServerClient = baseStore.lakehousePlatformServerClient;
this.ingestServerClient = baseStore.lakehouseIngestServerClient;
this.contractServerClient = baseStore.lakehouseContractServerClient;
this.graphManager = baseStore.graphManager;
this.taskService = baseStore.taskService;
this.alertService = baseStore.alertService;
this.layoutService = baseStore.layoutService;
this.aboutDisplay = this.layoutService.newDisplay(
'About',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 220,
x: -50,
y: 50,
center: false,
},
);
this.releaseLogDisplay = this.layoutService.newDisplay(
'Release Log',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 500,
},
);
this.releaseNotesDisplay = this.layoutService.newDisplay(
'Release Notes',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 350,
},
);
this.creator = new LegendDataCubeCreatorState(this);
this.loader = new LegendDataCubeLoaderState(this);
this.saverDisplay = new LegendDataCubeBlockingWindowState(
'Save DataCube',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 200,
},
);
this.codeEditorDisplay = new LegendDataCubeBlockingWindowState(
'Edit Latest Saved Query',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 400,
},
);
this.queryEditorDisplay = new LegendDataCubeBlockingWindowState(
'Edit Source Query',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 400,
},
);
this.deleteConfirmationDisplay = new LegendDataCubeBlockingWindowState(
'Delete DataCube',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 180,
},
);
this.sourceViewerDisplay = this.layoutService.newDisplay(
'DataCube Source',
() => ,
{
...DEFAULT_ALERT_WINDOW_CONFIG,
height: 200,
},
);
}
setBuilder(val: LegendDataCubeBuilderState | undefined) {
this.builder = val;
}
setSourceLoader(val: LegendDataCubeSourceLoaderState | undefined) {
this.sourceLoader = val;
}
private updateWindowTitle(persistentDataCube: PersistentDataCube) {
this.application.layoutService.setWindowTitle(
`\u229E ${persistentDataCube.name}${this.builder ? ` - ${formatDate(new Date(this.builder.startTime), 'HH:mm:ss EEE MMM dd yyyy')}` : ''}`,
);
}
getRecentlyViewedDataCubes() {
const data = this.application.userDataService.getObjectValue(
LegendDataCubeUserDataKey.RECENTLY_VIEWED_DATA_CUBES,
);
return data && Array.isArray(data) && data.every((id) => isString(id))
? data
: [];
}
canCurrentUserManageDataCube(
persistentDataCube: PersistentDataCube | LightPersistentDataCube,
) {
return (
persistentDataCube.owner === this.application.identityService.currentUser
);
}
async initialize() {
if (this.initializeState.isInProgress) {
return;
}
this.initializeState.inProgress();
try {
await this.engine.initialize();
this.initializeState.pass();
} catch (error) {
assertErrorThrown(error);
this.alertService.alertError(error, {
message: `Infrastructure Failure: ${error.message}`,
});
this.initializeState.fail();
}
}
async cleanUp() {
try {
await this.engine.dispose();
} catch (error) {
assertErrorThrown(error);
this.alertService.alertError(error, {
message: `Infrastructure Failure: ${error.message}`,
});
}
}
private getSourceLoader(
specification: DataCubeSpecification,
persistentDataCube: PersistentDataCube,
) {
const sourceData = specification.source;
const onSuccess = async () => {
const dataCubeId = persistentDataCube.id;
this.finalizeLoad(specification, persistentDataCube, dataCubeId);
this.loadState.pass();
};
const onError = async (error: unknown) => {
assertErrorThrown(error);
this.alertService.alertError(error, {
message: `DataCube Load Failure: ${error.message}`,
});
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(null),
);
this.loadState.fail();
};
switch (sourceData._type) {
case LOCAL_FILE_QUERY_DATA_CUBE_SOURCE_TYPE:
return new LocalFileDataCubeSourceLoaderState(
this.application,
this.engine,
this.alertService,
sourceData,
persistentDataCube,
onSuccess,
onError,
);
default:
throw new UnsupportedOperationError(
`Can't create source loader for unsupported type '${sourceData._type}'`,
);
}
}
async loadDataCube(dataCubeId: string | undefined) {
// internalize the parameters and clean them from the URL
const sourceData =
this.application.navigationService.navigator.getCurrentLocationParameterValue(
LEGEND_DATA_CUBE_ROUTE_PATTERN_TOKEN.SOURCE_DATA,
{ replaceUrlSafeBase64Characters: true },
);
if (sourceData && !dataCubeId) {
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(null),
);
// populate the creator if source data is specified
try {
await this.creator.finalize(JSON.parse(atob(sourceData)));
} catch (error) {
assertErrorThrown(error);
this.alertService.alertError(error, {
message: `DataCube Creation Failure: Can't materialize source from source data. Error: ${error.message}`,
});
this.setBuilder(undefined);
}
}
// When user just starts the application with no DataCube ID, and source data
if (!dataCubeId && !sourceData && !this.builder && !this.passedFirstLoad) {
this.creator.display.open();
}
this.passedFirstLoad = true;
if (dataCubeId !== this.builder?.persistentDataCube?.id) {
if (!dataCubeId) {
this.setBuilder(undefined);
return;
}
this.loadState.inProgress();
const task = this.taskService.newTask('Loading DataCube...');
try {
const persistentDataCube =
await this.baseStore.graphManager.getDataCube(dataCubeId);
const specification = DataCubeSpecification.serialization.fromJson(
persistentDataCube.content,
);
const sourceLoader = returnUndefOnError(() =>
this.getSourceLoader(specification, persistentDataCube),
);
if (sourceLoader !== undefined) {
this.setSourceLoader(sourceLoader);
sourceLoader.display.open();
return;
}
this.finalizeLoad(specification, persistentDataCube, dataCubeId);
this.loadState.pass();
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.LOAD_DATACUBE__SUCCESS,
{
...this.engine.getDataFromRawSource(specification.source),
dataCubeId: dataCubeId,
},
);
} catch (error) {
assertErrorThrown(error);
const message = `DataCube Load Failure: ${error.message}`;
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.LOAD_DATACUBE__FAILURE,
{
...this.engine.getDataFromRawSource(
this.builder?.initialSpecification.source,
),
dataCubeId: dataCubeId,
error: message,
},
);
this.alertService.alertError(error, {
message: message,
});
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(null),
);
this.loadState.fail();
} finally {
this.taskService.endTask(task);
}
}
}
private finalizeLoad(
specification: DataCubeSpecification,
persistentDataCube: PersistentDataCube,
dataCubeId: string,
) {
this.setBuilder(
new LegendDataCubeBuilderState(this, specification, persistentDataCube),
);
this.updateWindowTitle(persistentDataCube);
// update the list of stack of recently viewed DataCubes
const recentlyViewedDataCubes = this.getRecentlyViewedDataCubes();
const idx = recentlyViewedDataCubes.findIndex(
(data) => data === dataCubeId,
);
if (idx === -1) {
if (recentlyViewedDataCubes.length >= RECENTLY_VIEWED_DATA_CUBES_LIMIT) {
recentlyViewedDataCubes.pop();
}
recentlyViewedDataCubes.unshift(dataCubeId);
} else {
recentlyViewedDataCubes.splice(idx, 1);
recentlyViewedDataCubes.unshift(dataCubeId);
}
this.application.userDataService.persistValue(
LegendDataCubeUserDataKey.RECENTLY_VIEWED_DATA_CUBES,
recentlyViewedDataCubes,
);
}
private async generatePersistentDataCube(
api: DataCubeAPI,
name: string,
existingPersistentDataCube?: PersistentDataCube | undefined,
options?: LegendDataCubeSaveOptions,
) {
const specification = await api.generateSpecification();
let persistentDataCube: PersistentDataCube;
if (existingPersistentDataCube) {
persistentDataCube = existingPersistentDataCube.clone();
} else {
persistentDataCube = new PersistentDataCube();
persistentDataCube.id = uuid();
}
if (options !== undefined) {
specification.options =
specification.options ?? new DataCubeSpecificationOptions();
specification.options.autoEnableCache = options.autoEnableCache;
if (options.syncName && specification.configuration) {
specification.configuration.name = name;
}
}
persistentDataCube.name = name;
persistentDataCube.content =
DataCubeSpecification.serialization.toJson(specification);
return persistentDataCube;
}
async createNewDataCube(name: string, options?: LegendDataCubeSaveOptions) {
if (!this.builder?.dataCube || this.saveState.isInProgress) {
return;
}
this.saveState.inProgress();
const task = this.taskService.newTask('Creating DataCube...');
try {
const persistentDataCube = await this.generatePersistentDataCube(
this.builder.dataCube,
name,
undefined,
options,
);
const newPersistentDataCube =
await this.baseStore.graphManager.createDataCube(persistentDataCube);
// NOTE: reload is the cleanest, least bug-prone handling here
// but we can opt for just updating the URL to reflect the new DataCube
// as an optimization. Also, it helps preserve the edition history
// on the existing data-cube.
//
// Another way to avoid reloading the whole app it to force update
// the component using the key prop that ties to an ID
// of the builder.
this.builder.setPersistentDataCube(newPersistentDataCube);
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(newPersistentDataCube.id),
);
this.updateWindowTitle(persistentDataCube);
if (options?.syncName) {
this.builder.dataCube.updateName(name);
}
this.saverDisplay.close();
this.saveState.pass();
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.CREATE_DATACUBE__SUCCESS,
this.engine.getDataFromSource(
this.builder.dataCube.getProcessedSource(),
),
);
} catch (error) {
assertErrorThrown(error);
const message = `DataCube Creation Failure: ${error.message}`;
this.alertService.alertError(error, {
message: message,
});
this.saveState.fail();
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.CREATE_DATACUBE__FAILURE,
{
...this.engine.getDataFromSource(
this.builder.dataCube.getProcessedSource(),
),
error: message,
},
);
} finally {
this.taskService.endTask(task);
}
}
async saveDataCube(
name: string,
options?: LegendDataCubeSaveOptions & { saveAsNew?: boolean | undefined },
) {
if (!this.builder?.dataCube || this.saveState.isInProgress) {
return;
}
this.saveState.inProgress();
const task = this.taskService.newTask('Saving DataCube...');
try {
const persistentDataCube = await this.generatePersistentDataCube(
this.builder.dataCube,
name,
this.builder.persistentDataCube,
options,
);
if (options?.saveAsNew) {
persistentDataCube.id = uuid();
const newPersistentDataCube =
await this.baseStore.graphManager.createDataCube(persistentDataCube);
// NOTE: reload is the cleanest, least bug-prone handling here
// but we can opt for just updating the URL to reflect the new DataCube
// as an optimization. Also, it helps preserve the edition history
// on the existing data-cube.
//
// Another way to avoid reloading the whole app it to force update
// the component using the key prop that ties to an ID
// of the builder.
this.builder.setPersistentDataCube(newPersistentDataCube);
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(newPersistentDataCube.id),
);
} else {
const updatedPersistentDataCube =
await this.baseStore.graphManager.updateDataCube(persistentDataCube);
this.builder.setPersistentDataCube(updatedPersistentDataCube);
}
this.updateWindowTitle(persistentDataCube);
if (options?.syncName) {
this.builder.dataCube.updateName(name);
}
this.saverDisplay.close();
this.saveState.pass();
this.engine.sendTelemetry(
options?.saveAsNew
? LEGEND_DATACUBE_APP_EVENT.CREATE_DATACUBE__SUCCESS
: LEGEND_DATACUBE_APP_EVENT.UPDATE_DATACUBE__SUCCESS,
{
...this.engine.getDataFromSource(
this.builder.dataCube.getProcessedSource(),
),
autoEnableCache: !!options?.autoEnableCache,
},
);
} catch (error) {
assertErrorThrown(error);
const message = `DataCube Update Failure: ${error.message}`;
this.engine.sendTelemetry(
options?.saveAsNew
? LEGEND_DATACUBE_APP_EVENT.CREATE_DATACUBE__FAILURE
: LEGEND_DATACUBE_APP_EVENT.UPDATE_DATACUBE__FAILURE,
{
...this.engine.getDataFromSource(
this.builder.dataCube.getProcessedSource(),
),
autoEnableCache: !!options?.autoEnableCache,
error: message,
},
);
this.alertService.alertError(error, {
message: message,
});
this.saveState.fail();
} finally {
this.taskService.endTask(task);
}
}
setDataCubeToDelete(
val: LightPersistentDataCube | PersistentDataCube | undefined,
) {
this.dataCubeToDelete = val;
}
async updateBuilderWithNewSpecification(
state: LegendDataCubeCodeEditorState,
): Promise {
const builder = this.builder;
try {
if (
!builder ||
!(
builder.source instanceof LakehouseConsumerDataCubeSource ||
builder.source instanceof LakehouseProducerDataCubeSource
)
) {
throw new Error(
`DataCube builder is undefined or source type: '${builder?.source}' is not supported.`,
);
}
const newSource = builder.source;
let serializedRawSource;
if (newSource instanceof LakehouseConsumerDataCubeSource) {
const rawSource =
RawLakehouseConsumerDataCubeSource.serialization.fromJson(
builder.initialSpecification.source,
);
rawSource.query = state.code;
serializedRawSource =
RawLakehouseConsumerDataCubeSource.serialization.toJson(rawSource);
} else {
const rawSource =
RawLakehouseProducerDataCubeSource.serialization.fromJson(
builder.initialSpecification.source,
);
rawSource.query = state.code;
serializedRawSource =
RawLakehouseProducerDataCubeSource.serialization.toJson(rawSource);
}
const processedSource =
await this.engine.processSource(serializedRawSource);
const newSpecification = await this.engine.generateBaseSpecification(
serializedRawSource,
processedSource,
);
newSpecification.configuration =
builder.initialSpecification.configuration;
this.setBuilder(new LegendDataCubeBuilderState(this, newSpecification));
this.queryEditorDisplay.close();
} catch (error) {
assertErrorThrown(error);
const message = `DataCube reload Failure: ${error.message}`;
this.queryEditorDisplay.close();
this.alertService.alertError(error, {
message: message,
});
}
}
async deleteDataCube() {
if (this.deleteState.isInProgress || !this.dataCubeToDelete) {
return;
}
const dataCubeId = this.dataCubeToDelete.id;
this.deleteState.inProgress();
const task = this.taskService.newTask('Deleting DataCube...');
try {
const persistentDataCubeToDelete =
this.dataCubeToDelete instanceof PersistentDataCube
? this.dataCubeToDelete
: await this.baseStore.graphManager.getDataCube(dataCubeId);
const specification = DataCubeSpecification.serialization.fromJson(
persistentDataCubeToDelete.content,
);
await this.baseStore.graphManager.deleteDataCube(dataCubeId);
// update the list of stack of recently viewed DataCubes
const recentlyViewedDataCubes = this.getRecentlyViewedDataCubes();
const idx = recentlyViewedDataCubes.findIndex(
(data) => data === dataCubeId,
);
if (idx !== -1) {
recentlyViewedDataCubes.splice(idx, 1);
this.application.userDataService.persistValue(
LegendDataCubeUserDataKey.RECENTLY_VIEWED_DATA_CUBES,
recentlyViewedDataCubes,
);
}
if (this.builder?.persistentDataCube?.id === dataCubeId) {
this.application.navigationService.navigator.updateCurrentLocation(
generateBuilderRoute(null),
);
}
if (this.loader.selectedResult?.id === dataCubeId) {
this.loader.setSelectedResult(undefined);
}
this.setDataCubeToDelete(undefined);
this.deleteConfirmationDisplay.close();
this.deleteState.pass();
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.DELETE_DATACUBE__SUCCESS,
{
...this.engine.getDataFromRawSource(specification.source),
dataCubeDeletedId: dataCubeId,
},
);
} catch (error) {
assertErrorThrown(error);
const message = `DataCube Delete Failure: ${error.message}`;
this.engine.sendTelemetry(
LEGEND_DATACUBE_APP_EVENT.DELETE_DATACUBE__FAILURE,
{ error: message, dataCubeFailedToDeleteId: this.dataCubeToDelete.id },
);
this.alertService.alertError(error, {
message: message,
});
this.deleteState.fail();
} finally {
this.taskService.endTask(task);
}
}
}