/**
* @version 1.1
* @author Boris GBAHOUE
* @file Common logic for Mongo/Mongoose objects, abstraction layer to allow a simple switch from one DB to another
* @module amiwo/db/mongo
*/
// =======================================================================
// BASIC SETUP
// =======================================================================
const debug = require('debug')('amiwo:db');
const util = require('util');
const assert = require('assert');
const u = require('../../util');
const when = require('when');
// load our objects
const DBObject = require('../DBObject');
const DBObjectFactory = require('../DBObjectFactory');
const ObjectID = require('mongodb').ObjectID;
const GenericError = require('../../error/GenericError');
const ConcurrencyDBError = require('../../error/ConcurrencyDBError');
// class constants
const OCC_RETRIES = 1000;
// =======================================================================
// CONSTRUCTOR
// =======================================================================
/**
* Class variables are :
* - {mongoose.Schema} $document
* - {mongoose.Model} $model: used for db access functions
* - {String} $name: flag the property which should be used to find documents by name (as in findByName() method)
*
* @constructor
* @param {Array} linkedObjects: objects to be linked to this one i.e. removed when this object is removed
* @param {Object} [object]: to init this GenericMongoDBObject with
*
* @classdesc Common logic for Mongo/Mongoose objects, abstraction layer to allow a simple switch from one DB to another.
* @class
* @abstract
* @augments module:amiwo/db~DBObject
*/
function GenericMongoDBObject(linkedObjects, object) {
DBObject.call(this, linkedObjects, object);
this.$document = null; // will be initialized later
this.$name = null;
var $schemaHash = {}; // private variable
var $model;
Object.defineProperty(this, "$model", {
get: function() {
return $model;
},
set: function(value) {
$model = value;
if (value == null) return;
value.schema.eachPath($fillPath);
function $fillPath(pathName, schemaType) {
if (pathName.includes('.')) {
// nested path
let $split = pathName.split('.');
let $len = $split.length;
let $parent = $schemaHash;
for (let i = 0; i < $len; i++) {
if (i < ($len - 1)) {
if ($parent[$split[i]] == null) $parent[$split[i]] = {};
$parent = $parent[$split[i]];
} else {
if (schemaType.schema) {
// Embedeed document => go recursively
schemaType.schema.eachPath(function (subname, subtype) {
$fillPath(pathName+"."+subname, subtype);
});
} else {
$parent[$split[i]] = $analyseSchema(schemaType);
}
}
}
} else {
// simple path
$schemaHash[pathName] = $analyseSchema(schemaType);
if (schemaType.schema) {
// Embedeed document => go recursively
schemaType.schema.eachPath(function (subname, subtype) {
$fillPath(pathName+"."+subname, subtype);
});
}
}
} // end of $fillPath
}
});
Object.defineProperty(this, "schemaHash", {
enumerable: false,
get: function() {
return $schemaHash;
}
});
this.$model = null;
}
/**
* Inherit from `DBObject`.
*/
util.inherits(GenericMongoDBObject, DBObject);
// =======================================================================
// PROTECTED METHODS
// =======================================================================
/**
* Generic method to create an instance of Mongoose Document (new MongooseModel()) from
* this.$model. It expects 'this.$model' to have been set in the subclass's construtor
*
* @param {any} object
*
* @return {mongoose.Document}
*
* @protected
*/
GenericMongoDBObject.prototype.getDocument = function(object) {
assert(this.$model != null, "::AMIWO::"+this.constructor.name.toUpperCase()+"::GETDOCUMENT::ERROR 'this.$model' must be set before calling getDocument()");
return new this.$model(object);
}
// =======================================================================
// DATABASE ACCESS WRAPPERS
// =======================================================================
/**
* Encapsulate '$model.findByIdAndUpdate(id, query, options)'
*
* @param {string} id.id : id to lookup
* @param {string} query
* @param {object} [options]
*
* @return {Promise}
*
* @protected
*/
GenericMongoDBObject.prototype.dbFindOneAndUpdate = function (id, query, options) {
return $createQueryChain(this.$model.findByIdAndUpdate(id, query, options), options).exec();
};
/**
* Encapsulate 'this.$model.find(query)'
*
* @param {string} query : id to lookup
* @param {Object} [options]
*
* @return {Promise}
*
* @protected
*/
GenericMongoDBObject.prototype.dbFind = function (query, options) {
return $createQueryChain(this.$model.find(query), options).exec();
};
/**
* Encapsulate 'this.$model.findById(id)'
*
* @param {string} id : id to lookup
* @param {Object} [options]
*
* @return {Promise}
*
* @protected
*/
GenericMongoDBObject.prototype.dbFindById = function (id, options) {
return $createQueryChain(this.$model.findById(id), options).exec();
};
/**
* Encapsulate 'this.$model.findByIdAndRemove(id)'
*
* @param {Object|String|Number} id : id to lookup, if object structure as a Query
* @param {Object} [options]
*
* @return {Promise}
*
* @protected
*/
GenericMongoDBObject.prototype.dbFindByIdAndRemove = function (id, options) {
return this.$model.findByIdAndRemove(id, options).exec();
};
/**
* Encapsulate 'this.$model.remove(query)'
*
* @param {Object} [query] if empty remove the current object (this._id must then be set)
* @param {Object} [options]
*
* @return {Promise}
*
* @protected
*/
GenericMongoDBObject.prototype.dbRemove = function (query, options) {
// Prevent removing the whole Collection
if (u.isEmpty(query)) {
if (u.isEmpty(this._id)) return when.reject(new GenericError("::AMIWO::"+this.constructor.name.toUpperCase()+"::DBREMOVE::ERROR Query and this._id are empty, remove blocked"));
query = { "_id": this._id };
}
return this.$model.remove(query).exec();
};
/**
* Async save of 'this'
*
* @param {Object} options
* @param {boolean} [options.occ=false] set to true to force a concurrency check using Optimistic Concurrency Control
*
* @return {Promise} wrapping the updated object
*
* @protected
*/
GenericMongoDBObject.prototype.dbSave = function (options) {
var self = this;
if (options.occ === false) {
if (self._id == null) {
// New document
self.$document = self.getDocument(self.lean());
return self.$document.save();
} else {
// Update existing document
return self.$model.findByIdAndUpdate(self._id, self.lean(), {new: true}).exec(); // Return the new object
}
} else {
// Use __occ to ensure we have the proper version of the object
const $query = {
'_id': new ObjectID(self._id),
'__occ': self.__occ
};
const $tmp = self.lean();
const $update = {
'$set': {},
'$inc': {'__occ': 1}
};
for (let key in $tmp) {
if (["__occ", "_id"].includes(key)) continue;
$update['$set'][key] = self.$convertToObjectID($tmp[key], key);
}
return self.$updateOCC($query, $update, {returnOriginal: false});
}
};
/**
* Adds a value to an array unless the value is already present, in which case it does nothing to that array. The underlying DBObject is saved
*
* @param {String} property: name of the array property
* @param {any} value
* @param {Object} options
* @param {boolean} [options.each=true] set to true to add all elements of 'value' separately. If set to false and 'value' is an array adds the Array
* @param {boolean} [options.slice] same as $slice operator in Mongo see https://docs.mongodb.com/manual/reference/operator/update/slice/
*
* @return {Promise} wrapping the value which were added
*/
GenericMongoDBObject.prototype.dbAddToSet = function(property, value, options) {
const self = this;
// Convert into proper ObjectId objects
let $convertedValues = self.$convertToObjectID(value, property);
let $value = {};
let $slice = isNaN(Number(options.slice)) ? false : Number(options.slice);
if (options.each) {
if ($slice) {
$value[property] = {
"$each": $convertedValues,
"$slice": $slice
};
} else {
$value[property] = {
"$each": $convertedValues
};
}
} else {
if ($slice) {
$value[property] = {
"$each": [ $convertedValues ],
"$slice": $slice
}
} else {
$value[property] = $convertedValues;
}
}
var $update = {
'$addToSet': $value
};
// Figure out which elements will be added to the set (assuming we are in sync with DB)
let $property = u.getProperty(self, property);
var addedElements = u.notInSet($property, $convertedValues);
if (options.occ === false) {
return self.$model.findByIdAndUpdate(self._id, $update, {new: true}).exec()
.then(function(updatedObject) {
if (updatedObject == null) return when.resolve(updatedObject);
return when.resolve(addedElements);
})
} else {
// Use __occ to ensure we have the proper version of the object
let $query = {
'_id': new ObjectID(self._id),
'__occ': self.__occ
};
$update['$inc'] = {'__occ': 1};
return self.$updateOCC($query, $update, {returnOriginal: false}, addedElements);
}
}
/**
* Appends a specified value to an array. The underlying DBObject is saved
*
* @param {String} property: name of the array property
* @param {any} value
* @param {Object} options
* @param {boolean} [options.each=true] set to true to add all elements of 'value' separately. If set to false and 'value' is an array adds the Array
*
* @return {Promise} wrapping the value which were added
*/
GenericMongoDBObject.prototype.dbPush = function(property, value, options) {
const self = this;
// Convert into proper ObjectId objects
let $convertedValues = self.$convertToObjectID(value, property);
// Create $value object
var $values = {};
let $slice = isNaN(Number(options.slice)) ? false : Number(options.slice);
if (options.each) {
if ($slice) {
$values[property] = {
"$each": $convertedValues,
"$slice": $slice
};
} else {
$values[property] = {
"$each": $convertedValues
};
}
} else {
if ($slice) {
$values[property] = {
"$each": [ $convertedValues ],
"$slice": $slice
}
} else {
$values[property] = $convertedValues;
}
}
var $update = {
"$push": $values
};
if (options.occ === false) {
return self.$model.findByIdAndUpdate(self._id, $update, {new: true}).exec();
} else {
var $query = {
'_id': new ObjectID(self._id),
'__occ': self.__occ
}
$update['$inc'] = {'__occ': 1};
return self.$updateOCC($query, $update, {returnOriginal: false});
}
}
/**
* Replaces all document references by the actual DBObject
*
* @param {String[]|String|Object|Object[]} [path] path or array of path to populate; if nully populate all available references; if Object or [Object] uses Mongoose deep populate (see http://mongoosejs.com/docs/populate.html#deep-populate)
* @return {Promise} wrapping the populated DBObject
*
* @protected
*/
GenericMongoDBObject.prototype.dbPopulate = function(path) {
var self = this;
// Update $document
self.$document = self.getDocument(self.lean());
//var schema = self.$model.schema;
var modelHash = {};
var populateString = ""; // In mongoose >= 3.6, we can pass a space delimited string of path names to populate
var populatePromise;
// Triage based on path type
function $triage(obj) {
if (u.isEmpty(obj)) {
return 1;
} else if (Array.isArray(obj)) {
return $triage(obj[0]);
} else {
if (u.typeOf(obj) == "string") {
return 1;
} else if ((u.typeOf(obj) == "object") && obj.hasOwnProperty("path")) {
return 2;
} else {p
return -1;
}
}
}
let triage = $triage(path);
if (triage === 1) {
if (u.isEmpty(path)) {
// Populate all available paths
for (var key in self.$document.toObject()) {
if (self.$document[key] == null) continue; // Don't populate empty properties
if (typeof self.$document[key] === 'function') continue;
// var ref = self.$getDocumentReferenced(schema.path(key));
var ref = self.$getDocumentReferenced(key);
if (u.isNotEmpty(ref)) {
if (u.typeOf(ref) === 'string') {
populateString += key+" ";
modelHash[key] = ref;
} else if (ref instanceof Object) {
for (var key2 in ref) {
populateString += key2+" ";
modelHash[key2] = ref[key2];
}
}
}
}
} else {
// Populate selected path(s)
var paths = Array.isArray(path) ? path : [path]; // for readability
for (var i = 0; i < paths.length; i++) {
if (u.getProperty(self.$document.toObject(), paths[i]) == null) continue; // Don't populate empty properties
// var ref = self.$getDocumentReferenced(u.getProperty(paths[i]));
var ref = self.$getDocumentReferenced(paths[i]);
if (u.isNotEmpty(ref)) {
if (u.typeOf(ref) === 'string') {
populateString += paths[i]+" ";
modelHash[paths[i]] = ref;
} else if (ref instanceof Object) {
for (var key in ref) {
populateString += key+" ";
modelHash[key] = ref[key];
}
}
}
}
}
populatePromise = self.$document.populate(populateString.trim()).execPopulate();
} else if (triage === 2) { // expect a {path, [model], [populate] :{path, [model], [populate]: {...}}} object (or array of such objects) as per mongoose spec
populatePromise = self.$document.populate(path).execPopulate()
.then(function(populatedDocument) {
function $convert(obj, params) {
if (u.hasOwnProperties(params, ["path", "model"])) { // if I don't have a model property I can't convert any further
let key = params.path;
let className = params.model;
if (params.hasOwnProperty("populate") && u.hasOwnProperties(params.populate, ["path", "model"])) {
if (Array.isArray(u.getProperty(obj, key))) {
for (var i = 0, len = u.getProperty(obj, key).length; i < len; i++) {
u.getProperty(obj, key)[i] = DBObjectFactory.createObject(className, $convert(u.getProperty(obj, key)[i], params.populate));
}
} else {
u.setProperty(obj, key, DBObjectFactory.createObject(className, $convert(u.getProperty(obj, key), params.populate)));
}
} else {
u.setProperty(obj, key, DBObjectFactory.createObject(className, u.getProperty(obj, key))); // Create a full fledged DBObject
}
}
return obj;
}
let $dbObj = self._convert(populatedDocument);
let paths = Array.isArray(path) ? path : [path];
for (var i = 0; i < paths.length; i++) {
$dbObj = $convert($dbObj, paths[i]); // Try to create nested DBObject structures using the path Object (using Mongoose spec)
}
return when.resolve($dbObj);
})
} else {
return when.reject(new GenericError(util.format("Invalid path type => path=%j and should be nully, String or String[], Object or [Object]", path)));
}
return populatePromise.then(function(populatedDocument) {
var dbObj = self._convert(populatedDocument);
var modifiedPaths = [];
for (var path in modelHash) {
// handles the creation of arrays when dbObj[path] is an Array
u.setProperty(dbObj, path, DBObjectFactory.createObject(modelHash[path], u.getProperty(dbObj, path)));
// remove null DBObjects which should have been populated (most likely the linked Document has been removed)
let $tmp = self.$dbPopulateCheckReference(dbObj, path);
if (u.isNotEmpty($tmp)) modifiedPaths = modifiedPaths.concat($tmp);
}
// If a save is needed do it asynchronously
if (u.isNotEmpty(modifiedPaths)) {
var toUpdate = {};
for (var i=0; i<modifiedPaths.length; i++) {
u.setProperty(toUpdate, modifiedPaths[i], null);
}
self.$model.findByIdAndUpdate(self._id, toUpdate, {new: true}).then(function(result) {
if (result == null) {
// Can occur if we are processing a remove request
return when.resolve(true);
}
debug("::AMIWO::%s::DBPOPULATE::LOG %s ID '%s' was updated to clean %j => these references to other Objects were removed", self.constructor.name.toUpperCase(), self.constructor.name.replace("Mongo",""), self._id, modifiedPaths);
return when.resolve(true);
}).catch(function(err) {
debug("::AMIWO::%s::DBPOPULATE::ERROR Unable to update %s ID '%s' to clean %j => these references to other Objects were removed", self.constructor.name.toUpperCase(), self.constructor.name.replace("Mongo",""), self._id, modifiedPaths);
return when.resolve(false);
})
}
return when.resolve(dbObj);
});
}
// =======================================================================
// PRIVATE METHODS
// =======================================================================
/**
* Check 'obj' to see if any of its paths need to be reset and return an array of path to set to null
*
* @param {Object} obj
* @param {String} path
*
* @return {String[]}
* @protected
*/
GenericMongoDBObject.prototype.$dbPopulateCheckReference = function(obj, path) {
var self = this;
if (u.isEmpty(obj)) return null;
var object = (path == null) ? obj : u.getProperty(obj, path);
if (u.isEmpty(object)) return null;
var modifiedPaths = [];
if (Array.isArray(object)) {
for (var i= 0; i < object.length; i++) {
modifiedPaths = modifiedPaths.concat(self.$dbPopulateCheckReference(object[i], null));
}
} else if (object.isEmpty()) {
// Log and clean our object in DB
debug("::AMIWO::%s::DBPOPULATE::LOG Unable to populate reference at path '%s' => removing it", self.constructor.name.toUpperCase(), path);
modifiedPaths.push(path);
}
return modifiedPaths;
}
/**
* Common logic for cFindXXX() methods to process the 'options' object passed in parameters and create a proper Query object
*
* @param {Promise} query
* @param {Object} options an object
* @returns {Promise}
* @memberOf GenericMongoDBObject
* @private
*/
function $createQueryChain(query, options) {
if (query == null) return when.reject(GenericError.create("::AMIWO::"+this.constructor.name.toUpperCase()+"::$CREATEQUERYCHAIN::ERROR Invalid query, it can't be null"));
if (options == null) options = {};
if (options.options) options = options.options;
// Process options
// - populateFields
if (u.isNotEmpty(options.populateFields)) {
query = query.populate(options.populateFields);
}
// - lean
if (options.lean == true) {
query = query.lean();
}
// - limit
if (!isNaN(Number(options.limit))) {
query = query.limit(Number(options.limit));
}
// - sort
if (u.isNotEmpty(options.sort)) {
query = query.sort(options.sort);
}
return query;
};
/**
* Tests a path to see if it references another Document.
* If so returned the model name(s) of the referenced Document(s); otherwise return null
*
* @param {String} path
*
* @returns {{path: String}|String|null}
* @protected
*/
GenericMongoDBObject.prototype.$getDocumentReferenced = function(path) {
var self = this;
if (u.isEmpty(path)) return null;
var descriptor = u.getProperty(self.schemaHash, path);
if (descriptor && (Object.keys(descriptor).length == 2) && u.hasOwnProperties(descriptor, ["ref", "nested"])) {
if (descriptor.ref) {
// Direct reference to a DBObject or array of references to other DBObjects
return descriptor.ref;
} else if (descriptor.nested) {
// Nested (sub)document
var result = {};
for (var key in descriptor.nested.object) {
let $tmp = self.$getDocumentReferenced(path+"."+key);
if ($tmp) result[path+"."+key] = $tmp;
}
return (u.isEmpty(result)) ? null : result;
} else {
return null;
}
} else if (descriptor instanceof Object) {
// Path is still a high level container => go recursively
var result = {};
for (var key in descriptor) {
let $tmp = self.$getDocumentReferenced(path+"."+key);
if (u.typeOf($tmp) === 'string') {
result[path+"."+key] = $tmp;
} else if ($tmp instanceof Object) {
result = u.merge(result, $tmp);
}
}
return (u.isEmpty(result)) ? null : result;
} else if (descriptor) {
// Shouldn't reach here
debug("::AMIWO::%s::$GETDOCUMENTREFENCED::LOG Invalid state processing %s for %j", self.constructor.name, path, self);
return null;
} else {
return null;
}
}
/**
* @protected
*/
GenericMongoDBObject.prototype.$convertToObjectID = function(value, path) {
const self = this;
if (u.isEmpty(value)) return value;
if (Array.isArray(value)) {
if (self.$getDocumentReferenced(path) === null) return value; // not an array of ref.
let $array = [];
value.forEach(function(element) {
$array.push(self.$convertToObjectID(element, path)); // path is the array's name
})
return $array;
} else if (value instanceof Object) {
let $output = {};
for (let key in value) {
$output[key] = self.$convertToObjectID(value[key], path+"."+key);
}
return $output;
} else {
let $isRef = (self.$getDocumentReferenced(path) !== null);
if ($isRef && (u.typeOf(value) == 'string')) {
return new ObjectID(value);
} else {
return value;
}
}
}
/**
* Tests a path to see if it references another Document.
*
* @param {String} path mongoose Path
*
* @returns {Object} JSON structured as
* {nested: false, ref: ""} => for a simple value
* {nested: false, ref: String} => for ObjectId referencing a 'ref' document
* {nested: {array: true}, ref: String} => for an array of 'ref' document
* {nested: {array: true, object: <similar JSON for each key>}} => for an array of subdocuments
* @memberOf GenericMongoDBObject
* @private
*/
function $analyseSchema(schema) {
if (u.isEmpty(schema)) return {nested: false, ref: ""};
if (schema.constructor.name === 'DocumentArray') { // Array of nested documents
var result = {
nested: {
array: true,
object: {}
}
};
schema.schema.eachPath(function(name, type) {
result.nested.object[name] = $analyseSchema(type);
});
return result;
} else if (schema.options.ref !== undefined) { // ObjectId referencing another Document}
return {nested: false, ref: schema.options.ref};
} else if (schema.caster && (schema.caster.options.ref !== undefined)) { // SchemaArray of ObjectId
return {nested: {array: true}, ref: schema.caster.options.ref};
} else {
return {nested: false, ref: ""};
}
}
/**
* Get the Mongo collection associated with this DBObject
*
* @return {mongo.Collection}
* @protected
*/
GenericMongoDBObject.prototype.$getMongoCollection = function() {
return DBObjectFactory.getDB().collection(this.$model.collection.collectionName); // Expect a Mongo database object
}
/**
* Perform an Optimistic Consistency Controled update operation
*
* @param {Object} query
* @param {Object} update
* @param {Object} [options] (see http://mongodb.github.io/node-mongodb-native/2.1/api/Collection.html#findOneAndUpdate for complete options list)
* @param {boolean} [options.returnOriginal=true] When false, returns the updated document rather than the original. The default is true.
* @param {boolean} [options.upsert=false] Upsert the document if it does not exist.
* @param {Object} [options.projection=null] Limits the fields to return for all matching documents.
* @param {Object} [options.sort=null] Determines which document the operation modifies if the query selects multiple documents.
* @param {any} [returnedObj] object to wrap in the Promise if fulfilled; if null return update's value (i.e. updated document or original document depending on 'options.returnOriginal')
*
* @return {Promise} wrapping 'returnedObj' / update's result
* @throws {ConcurrencyDBError}
* @protected
*/
GenericMongoDBObject.prototype.$updateOCC = function(query, update, options, returnedObj) {
const self = this;
return self.$getMongoCollection().findOneAndUpdate(query, update, options)
.then(function(r) {
var result = r.value;
if (result == null) {
// No object matching $query found => possible OCC error
// Try to load object
return self.$getMongoCollection().findOne({_id: new ObjectID(self._id)})
.then(function(obj) {
if (obj != null) {
// OCC error since we were able to find a document with this ID
return when.reject(new ConcurrencyDBError(util.format("::AMIWO::%s::$UPDATEOCC::ERROR Concurrency error => had __occ=%d while latest version has __occ=%d", self.constructor.name, self.__occ, obj.__occ), obj));
} else {
// No document with this ID => not an OCC error, return null
return when.resolve(obj);
}
});
}
// else Object was found and updated => return it
return when.resolve((returnedObj === undefined) ? result : returnedObj);
})
}
module.exports = GenericMongoDBObject;