import { config, redis } from "actionhero"; import Sequelize, { Op, WhereOptions } from "sequelize"; import { AfterDestroy, AfterSave, AllowNull, BeforeDestroy, BeforeSave, BelongsTo, Column, DataType, Default, DefaultScope, ForeignKey, HasMany, Is, Length, Table, } from "sequelize-typescript"; import { PropertyConfigurationObject } from "../classes/codeConfig"; import { APIData } from "../modules/apiData"; import { CLS } from "../modules/cls"; import { ConfigWriter } from "../modules/configWriter"; import { FilterHelper } from "../modules/filterHelper"; import { LockableHelper } from "../modules/lockableHelper"; import { PropertyOps } from "../modules/ops/property"; import { OptionHelper } from "../modules/optionHelper"; import { plugin } from "../modules/plugin"; import { StateMachine } from "../modules/stateMachine"; import { TopLevelGroupRules } from "../modules/topLevelGroupRules"; import { Filter } from "./Filter"; import { Group } from "./Group"; import { GroupRule } from "./GroupRule"; import { Mapping } from "./Mapping"; import { Option } from "./Option"; import { GrouparooRecord } from "./GrouparooRecord"; import { RecordProperty } from "./RecordProperty"; import { Run } from "./Run"; import { Source } from "./Source"; import { getGrouparooRunMode } from "../modules/runMode"; import { CommonModel } from "../classes/commonModel"; import { PropertiesCache } from "../modules/caches/propertiesCache"; import { SourcesCache } from "../modules/caches/sourcesCache"; const jsMap = { boolean: config?.sequelize?.dialect === "sqlite" ? "text" : "boolean", // there is no boolean type in SQLite date: "bigint", // we store things via timestamps in the DB email: "text", float: "float", integer: "bigint", phoneNumber: "text", string: "text", url: "text", } as const; export const propertyJSToSQLType = (jsType: keyof typeof jsMap) => jsMap[jsType]; export const PropertyTypes = [ "boolean", "date", "email", "float", "integer", "phoneNumber", "string", "url", ] as const; export interface SimplePropertyOptions extends OptionHelper.SimpleOptions {} const STATES = ["draft", "ready", "deleted"] as const; const STATE_TRANSITIONS = [ { from: "draft", to: "ready", checks: [(instance: Property) => instance.validateOptions()], }, { from: "draft", to: "deleted", checks: [] }, { from: "ready", to: "deleted", checks: [] }, { from: "deleted", to: "ready", checks: [(instance: Property) => instance.validateOptions()], }, ]; export interface PropertyFiltersWithKey extends FilterHelper.FiltersWithKey {} @DefaultScope(() => ({ where: { state: { [Op.notIn]: ["draft"] } }, })) @Table({ tableName: "properties", paranoid: false }) export class Property extends CommonModel { idPrefix() { return "prp"; } @Length({ min: 0, max: 191 }) @Default("") @Column key: string; @AllowNull(false) @Is("properType", (value) => { if (!PropertyTypes.includes(value)) { throw new Error(`${value} is not an allowed type`); } }) @Column(DataType.ENUM(...PropertyTypes)) type: typeof PropertyTypes[number]; @AllowNull(false) @Default(false) @Column unique: boolean; @AllowNull(false) @ForeignKey(() => Source) @Column sourceId: string; @AllowNull(false) @Default("draft") @Column(DataType.ENUM(...STATES)) state: typeof STATES[number]; @Column locked: string; @AllowNull(false) @Default(false) @Column isPrimaryKey: boolean; @AllowNull(false) @Default(false) @Column isArray: boolean; @BelongsTo(() => Source) source: Source; @HasMany(() => Option, { foreignKey: "ownerId", scope: { ownerType: "property" }, }) __options: Option[]; // the underscores are needed as "options" is an internal method on sequelize instances @HasMany(() => Filter, { foreignKey: "ownerId", scope: { ownerType: "property" }, }) filters: Filter[]; @HasMany(() => RecordProperty) RecordProperties: RecordProperty[]; async parameterizedQueryFromRecord(q: string, record: GrouparooRecord) { return plugin.replaceTemplateRecordVariables(q, record); } async test(options?: SimplePropertyOptions) { const record = await GrouparooRecord.findOne({ order: [["id", "asc"]], }); if (record) { const source = await Source.findById(this.sourceId); return source.importRecordProperty(record, this, options); } } async getOptions(sourceFromEnvironment = true) { const options = await OptionHelper.getOptions(this, sourceFromEnvironment); const source = await SourcesCache.findOneWithCache(this.sourceId); for (const i in options) { options[i] = await plugin.replaceTemplateRecordPropertyIdsWithRecordPropertyKeys( options[i].toString(), source.modelId ); } return options; } async setOptions( options: SimplePropertyOptions, test = true, externallyValidate = true ) { if (test) await this.test(options); const source = await SourcesCache.findOneWithCache(this.sourceId); for (const i in options) { options[i] = await plugin.replaceTemplateRecordPropertyKeysWithRecordPropertyId( options[i].toString(), source.modelId ); } return OptionHelper.setOptions(this, options, externallyValidate); } async afterSetOptions(hasChanges: boolean) { if (hasChanges) { await Property.invalidateCache(); await PropertyOps.enqueueRuns(this); } } async validateOptions( options?: SimplePropertyOptions, externallyValidate = true ) { if (!options) options = await this.getOptions(true); if (!externallyValidate) return; const pluginOptions = await this.pluginOptions(options); const optionsSpec: OptionHelper.OptionsSpec = pluginOptions.map((opt) => ({ ...opt, options: opt.options?.map((o) => o.key), })); await OptionHelper.validateOptions(this, options, optionsSpec); } async getPlugin() { return OptionHelper.getPlugin(this); } async getFilters() { return FilterHelper.getFilters(this); } async setFilters( filters: PropertyFiltersWithKey[], externallyValidate = true ) { return FilterHelper.setFilters(this, filters, externallyValidate); } async afterSetFilters(hasChanges: boolean) { if (hasChanges) { await Property.invalidateCache(); return PropertyOps.enqueueRuns(this); } } async pluginOptions(propertyOptions?: SimplePropertyOptions) { return PropertyOps.pluginOptions(this, propertyOptions); } async validateFilters(filters: PropertyFiltersWithKey[]) { return FilterHelper.validateFilters(this, filters); } async apiData() { const options = await this.getOptions(); const filters = await this.getFilters(); return { id: this.id, sourceId: this.sourceId, key: this.key, type: this.type, state: this.state, unique: this.unique, isPrimaryKey: this.isPrimaryKey, locked: this.locked, options, filters, isArray: this.isArray, createdAt: APIData.formatDate(this.createdAt), updatedAt: APIData.formatDate(this.updatedAt), recordProperties: this.RecordProperties ? await Promise.all(this.RecordProperties.map((rp) => rp.apiData())) : undefined, }; } getConfigId() { return this.idIsDefault() ? ConfigWriter.generateId(this.key) : this.id; } async getConfigObject(): Promise { const { key, type, unique, isArray } = this; this.source = await this.$get("source"); const sourceId = this.source?.getConfigId(); const options = await this.getOptions(false); const filters = await this.getFilters(); if (!key || !sourceId) return; return { class: "Property", id: this.getConfigId(), type, key, sourceId, unique, isArray, options, filters, }; } // --- Class Methods --- // @BeforeSave static async ensureUniquePrimaryKey(instance: Property) { if (instance.isPrimaryKey && !instance.unique) { throw new Error( `Property "${instance.key}" must be unique because it‘s the model‘s Primary Key.` ); } } @BeforeSave static async ensureOptions(instance: Property) { const source = await Source.findById(instance.sourceId); await source.validateOptions(null); } @BeforeSave static async updateState(instance: Property) { await StateMachine.transition(instance, STATE_TRANSITIONS); } @BeforeSave static async ensureNonArrayAndUnique(instance: Property) { if (instance.isArray && instance.unique) { throw new Error("unique record properties cannot be arrays"); } } @BeforeSave static async ensureUniqueProperties(instance: Property) { if (instance.changed("unique") && instance.unique) { const valueCounts = await RecordProperty.findAll({ attributes: [ "rawValue", [Sequelize.fn("COUNT", Sequelize.col("rawValue")), "count"], ], group: ["rawValue"], where: { propertyId: instance.id }, having: Sequelize.where( Sequelize.fn("COUNT", Sequelize.col("rawValue")), { [Op.gt]: 1 } ), limit: 1, raw: true, }); if (valueCounts.length > 0) { throw new Error( //@ts-ignore `cannot make this property unique as there are ${valueCounts[0]["count"]} records with the value \'${valueCounts[0]["rawValue"]}\'` ); } } } @BeforeSave static async ensureSourceReady(instance: Property) { const source = await Source.findById(instance.sourceId); if (source.state !== "ready") { // allow the bootstrap for this source (first one and unique) if (!instance.unique) { throw new Error("source is not ready and property not unique"); } const otherProperties = await Property.scope(null).count({ where: { sourceId: instance.sourceId, id: { [Op.ne]: instance.id }, }, }); if (otherProperties > 0) { // already has another property throw new Error("source is not ready and property not first"); } } } @BeforeSave static async validateQuery(instance: Property) { await instance.test(); } @BeforeSave static async validateReservedKeys(instance: Property) { const reservedKeys = ["_meta"].concat( TopLevelGroupRules.map((tlgr) => tlgr.key) ); if (reservedKeys.includes(instance.key)) { throw new Error( `${instance.key} is a reserved key and cannot be used as a property` ); } } @BeforeSave static async noUniqueOrArrayThroughNonUniqueMapping(instance: Property) { if (instance.state === "draft") return; const source = await Source.findById(instance.sourceId); if (source.state !== "ready") return; // we are bootstrapping const sourceMapping = await source.getMapping(); if (Object.keys(sourceMapping).length === 0) return; // Query source const mappedPropertyKey = Object.values(sourceMapping)[0]; const mappedProperty = await Property.findOne({ where: { key: mappedPropertyKey }, }); if (mappedProperty.unique) return; if (instance.unique) throw new Error( `Unique Property ${instance.key} (${instance.id}) cannot be mapped through a non-unique Property - ${mappedProperty.key} (${mappedProperty.id})` ); } @BeforeSave static async noUpdateIfLocked(instance: Property) { await LockableHelper.beforeSave(instance, ["state", "isPrimaryKey"]); } @BeforeSave static async runPropertyWithNoOptionsOrFiltersWhenReady(instance: Property) { const changedState = instance.changed("state"); if (changedState && instance.state === "ready") { const options = await instance.getOptions(); const filters = await instance.getFilters(); if ( Object.keys(options).length === 0 && Object.keys(filters).length === 0 ) { await PropertyOps.enqueueRuns(instance); } } } @BeforeSave static async updateRecordPropertyUniqueness(instance: Property) { if (!instance.isNewRecord && instance.changed("unique")) { CLS.enqueueTask("property:updateRecordProperties", { propertyId: instance.id, }); } } @BeforeDestroy static async ensureNotInUse( instance: Property, excludeMappingOwnerIds?: string[] ) { const groupRule = await GroupRule.findOne({ where: { propertyId: instance.id }, }); if (groupRule) { const group = await Group.findById(groupRule.groupId); throw new Error( `cannot delete property "${instance.key}", group ${group.name} (${group.id}) is based on it` ); } const mappingWhere: WhereOptions = { propertyId: instance.id }; if (excludeMappingOwnerIds) { mappingWhere.ownerId = { [Op.notIn]: excludeMappingOwnerIds }; } const mapping = await Mapping.findOne({ where: mappingWhere, }); if (mapping) { throw new Error( `cannot delete property "${instance.key}" as ${mapping.ownerId} is using it in a mapping` ); } } @AfterSave static async updateRecordsOnTypeChange(instance: Property) { if (instance.changed("type") && instance.state === "ready") { await PropertyOps.enqueueRuns(instance); } } @AfterSave static async updateSampleRecords(instance: Property) { if (getGrouparooRunMode() !== "cli:config") return; if (instance.state !== "ready") return; const source = await instance.$get("source"); if (!source) return; const records = await GrouparooRecord.findAll({ where: { modelId: source.modelId }, }); for (const record of records) await record.buildNullProperties(); } @BeforeDestroy static async noDestroyIfLocked(instance: Property) { await LockableHelper.beforeDestroy(instance); } @AfterDestroy static async destroyOptions(instance: Property) { await Option.destroy({ where: { ownerId: instance.id, ownerType: "property" }, }); } @AfterDestroy static async stopRuns(instance: Property) { const runs = await Run.findAll({ where: { creatorId: instance.id, state: "running" }, }); for (const i in runs) { await runs[i].update({ state: "stopped" }); } } @AfterDestroy static async destroyFilters(instance: Property) { await Filter.destroy({ where: { ownerId: instance.id, ownerType: "property" }, }); } @AfterDestroy static async destroyRecordProperties(instance: Property) { await RecordProperty.destroy({ where: { propertyId: instance.id }, }); } @AfterSave @AfterDestroy static async invalidateCache() { PropertiesCache.invalidate(); await CLS.afterCommit( async () => await redis.doCluster("api.rpc.property.invalidateCache") ); } }