/**
* Module for some common utility functions and references
* @module api/utils/common
*/
/** @lends module:api/utils/common */
var common = {},
moment = require('moment-timezone'),
crypto = require('crypto'),
logger = require('./log.js'),
mcc_mnc_list = require('mcc-mnc-list'),
plugins = require('../../plugins/pluginManager.js'),
countlyConfig = require('./../config', 'dont-enclose'),
argon2 = require('argon2');
var matchHtmlRegExp = /"|'|&(?!amp;|quot;|#39;|lt;|gt;|#46;|#36;)|<|>/;
var matchLessHtmlRegExp = /[<>]/;
/**
* Escape special characters in the given string of html.
* @param {string} string - The string to escape for inserting into HTML
* @param {bool} more - if false, escapes only tags, if true escapes also quotes and ampersands
* @returns {string} escaped string
**/
common.escape_html = function(string, more) {
var str = '' + string;
var match;
if (more) {
match = matchHtmlRegExp.exec(str);
}
else {
match = matchLessHtmlRegExp.exec(str);
}
if (!match) {
return str;
}
var escape;
var html = '';
var index = 0;
var lastIndex = 0;
for (index = match.index; index < str.length; index++) {
switch (str.charCodeAt(index)) {
case 34: // "
escape = '"';
break;
case 38: // &
escape = '&';
break;
case 39: // '
escape = ''';
break;
case 60: // <
escape = '<';
break;
case 62: // >
escape = '>';
break;
default:
continue;
}
if (lastIndex !== index) {
html += str.substring(lastIndex, index);
}
lastIndex = index + 1;
html += escape;
}
return lastIndex !== index ? html + str.substring(lastIndex, index) : html;
};
/**
* Decode escaped html
* @param {string} string - The string to decode
* @returns {string} escaped string
**/
common.decode_html = function(string) {
string = string.replace(/&/g, '&');
string = string.replace(/'/g, "'");
string = string.replace(/"/g, '"');
string = string.replace(/</g, '<');
string = string.replace(/>/g, '>');
return string;
};
/**
* Escape special characters in the given value, may be nested object
* @param {string} key - key of the value
* @param {vary} value - value to escape
* @param {bool} more - if false, escapes only tags, if true escapes also quotes and ampersands
* @returns {vary} escaped value
**/
function escape_html_entities(key, value, more) {
if (typeof value === 'object' && value && (value.constructor === Object || value.constructor === Array)) {
if (Array.isArray(value)) {
let replacement = [];
for (let k = 0; k < value.length; k++) {
if (typeof value[k] === "string") {
let ob = getJSON(value[k]);
if (ob.valid) {
replacement[common.escape_html(k, more)] = JSON.stringify(escape_html_entities(k, ob.data, more));
}
else {
replacement[k] = common.escape_html(value[k], more);
}
}
else {
replacement[k] = escape_html_entities(k, value[k], more);
}
}
return replacement;
}
else {
let replacement = {};
for (let k in value) {
if (Object.hasOwnProperty.call(value, k)) {
if (typeof value[k] === "string") {
let ob = getJSON(value[k]);
if (ob.valid) {
replacement[common.escape_html(k, more)] = JSON.stringify(escape_html_entities(k, ob.data, more));
}
else {
replacement[common.escape_html(k, more)] = common.escape_html(value[k], more);
}
}
else {
replacement[common.escape_html(k, more)] = escape_html_entities(k, value[k], more);
}
}
}
return replacement;
}
}
return value;
}
/**
* Check if string is a valid json
* @param {string} val - string that might be json encoded
* @returns {object} with property data for parsed data and property valid to check if it was valid json encoded string or not
**/
function getJSON(val) {
var ret = {valid: false};
try {
ret.data = JSON.parse(val);
if (ret.data && typeof ret.data === "object") {
ret.valid = true;
}
}
catch (ex) {
//silent error
}
return ret;
}
/**
* Logger object for creating module specific logging
* @type {module:api/utils/log~Logger}
* @example
* var log = common.log('myplugin:api');
* log.i('myPlugin got a request: %j', params.qstring);
*/
common.log = logger;
/**
* Mapping some common property names from longer understandable to shorter representation stored in database
* @type {object}
*/
common.dbMap = {
'events': 'e',
'total': 't',
'new': 'n',
'unique': 'u',
'duration': 'd',
'durations': 'ds',
'frequency': 'f',
'loyalty': 'l',
'sum': 's',
'dur': 'dur',
'count': 'c'
};
/**
* Mapping some common user property names from longer understandable to shorter representation stored in database
* @type {object}
*/
common.dbUserMap = {
'device_id': 'did',
'user_id': 'uid',
'first_seen': 'fs',
'last_seen': 'ls',
'last_payment': 'lp',
'session_duration': 'sd',
'total_session_duration': 'tsd',
'session_count': 'sc',
'device': 'd',
'device_type': 'dt',
'carrier': 'c',
'city': 'cty',
'region': 'rgn',
'country_code': 'cc',
'platform': 'p',
'platform_version': 'pv',
'app_version': 'av',
'last_begin_session_timestamp': 'lbst',
'last_end_session_timestamp': 'lest',
'has_ongoing_session': 'hos',
'previous_events': 'pe',
'resolution': 'r'
};
common.dbUniqueMap = {
"*": [common.dbMap.unique],
users: [common.dbMap.unique, common.dbMap.durations, common.dbMap.frequency, common.dbMap.loyalty]
};
/**
* Mapping some common event property names from longer understandable to shorter representation stored in database
* @type {object}
*/
common.dbEventMap = {
'user_properties': 'up',
'timestamp': 'ts',
'segmentations': 'sg',
'count': 'c',
'sum': 's',
'duration': 'dur',
'previous_events': 'pe'
};
/**
* Default {@link countlyConfig} object for API server
* @type {object}
*/
common.config = countlyConfig;
/**
* Reference to momentjs
* @type {object}
*/
common.moment = moment;
/**
* Reference to crypto module
* @type {object}
*/
common.crypto = crypto;
/**
* Operating syste/platform mappings from what can be passed in metrics to shorter representations
* stored in db as prefix to OS segmented values
* @type {object}
*/
common.os_mapping = {
"webos": "webos",
"brew": "brew",
"unknown": "unk",
"undefined": "unk",
"tvos": "atv",
"apple tv": "atv",
"watchos": "wos",
"unity editor": "uty",
"qnx": "qnx",
"os/2": "os2",
"amazon fire tv": "aft",
"amazon": "amz",
"web": "web",
"windows": "mw",
"open bsd": "ob",
"searchbot": "sb",
"sun os": "so",
"solaris": "so",
"beos": "bo",
"mac osx": "o",
"macos": "o",
"mac": "o",
"osx": "o",
"linux": "l",
"unix": "u",
"ios": "i",
"android": "a",
"blackberry": "b",
"windows phone": "w",
"wp": "w",
"roku": "r",
"symbian": "s",
"chrome": "c",
"debian": "d",
"nokia": "n",
"firefox": "f",
"tizen": "t"
};
/**
* Whole base64 alphabet for fetching splitted documents
* @type {object}
*/
common.base64 = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "+", "/"];
common.dbPromise = function() {
var args = Array.prototype.slice.call(arguments);
return new Promise(function(resolve, reject) {
var collection = common.db.collection(args[0]),
method = args[1];
if (method === 'find') {
collection[method].apply(collection, args.slice(2)).toArray(function(err, result) {
if (err) {
reject(err);
}
else {
resolve(result);
}
});
}
else {
collection[method].apply(collection, args.slice(2).concat([function(err, result) {
if (err) {
reject(err);
}
else {
resolve(result);
}
}]));
}
});
};
/**
* Fetches nested property values from an obj.
* @param {object} obj - standard countly metric object
* @param {string} desc - dot separate path to fetch from object
* @returns {object} fetched object from provided path
* @example
* //outputs {"u":20,"t":20,"n":5}
* common.getDescendantProp({"2017":{"1":{"2":{"u":20,"t":20,"n":5}}}}, "2017.1.2");
*/
common.getDescendantProp = function(obj, desc) {
desc = String(desc);
if (desc.indexOf(".") === -1) {
return obj[desc];
}
var arr = desc.split(".");
while (arr.length && (obj = obj[arr.shift()])) {
//doing operator in the loop condition
}
return obj;
};
/**
* Checks if provided value could be converted to a number,
* even if current type is other, as string, as example value "42"
* @param {any} n - value to check if it can be converted to number
* @returns {boolean} true if can be a number, false if can't be a number
* @example
* common.isNumber(1) //outputs true
* common.isNumber("2") //outputs true
* common.isNumber("test") //outputs false
*/
common.isNumber = function(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
};
/**
* This default Countly behavior of type conversion for storing proeprties accepted through API requests
* dealing with numbers as strings and too long numbers
* @param {any} value - value to convert to usable type
* @returns {varies} converted value
* @example
* common.convertToType(1) //outputs 1
* common.convertToType("2") //outputs 2
* common.convertToType("test") //outputs "test"
* common.convertToType("12345678901234567890") //outputs "12345678901234567890"
*/
common.convertToType = function(value) {
//handle array values
if (Array.isArray(value)) {
for (var i = 0; i < value.length; i++) {
value[i] = common.convertToType(value[i]);
}
return value;
}
else if (value && typeof value === "object") {
for (var key in value) {
value[key] = common.convertToType(value[key]);
}
return value;
}
//if value can be a number
else if (common.isNumber(value)) {
//check if it is string but is less than 16 length
if (value.length && value.length <= 16) {
//convert to number
return parseFloat(value);
}
//check if it is number, but longer than 16 digits (max limit)
else if ((Math.round(value) + "").length > 16) {
//convert to string
return value + "";
}
else {
//return number as is
return value;
}
}
else {
//return as string
return value + "";
}
};
/**
* Safe division between numbers providing 0 as result in cases when dividing by 0
* @param {number} dividend - number which to divide
* @param {number} divisor - number by which to divide
* @returns {number} result of division
* @example
* //outputs 0
* common.safeDivision(100, 0);
*/
common.safeDivision = function(dividend, divisor) {
var tmpAvgVal;
tmpAvgVal = dividend / divisor;
if (!tmpAvgVal || tmpAvgVal === Number.POSITIVE_INFINITY) {
tmpAvgVal = 0;
}
return tmpAvgVal;
};
/**
* Pad number with specified character from left to specified length
* @param {number} number - number to pad
* @param {number} width - pad to what length in symbols
* @returns {string} padded number
* @example
* //outputs 0012
* common.zeroFill(12, 4, "0");
*/
common.zeroFill = function(number, width) {
width -= number.toString().length;
if (width > 0) {
return new Array(width + (/\./.test(number) ? 2 : 1)).join('0') + number;
}
return number + ""; // always return a string
};
/**
* Add item or array to existing array only if values are not already in original array
* @param {array} arr - original array where to add unique elements
* @param {string|number|array} item - item to add or array to merge
*/
common.arrayAddUniq = function(arr, item) {
if (!arr) {
arr = [];
}
if (toString.call(item) === "[object Array]") {
for (var i = 0; i < item.length; i++) {
if (arr.indexOf(item[i]) === -1) {
arr[arr.length] = item[i];
}
}
}
else {
if (arr.indexOf(item) === -1) {
arr[arr.length] = item;
}
}
};
/**
* Create HMAC sha1 hash from provided value and optional salt
* @param {string} str - value to hash
* @param {string=} addSalt - optional salt, uses ms timestamp by default
* @returns {string} HMAC sha1 hash
*/
common.sha1Hash = function(str, addSalt) {
var salt = (addSalt) ? new Date().getTime() : '';
return crypto.createHmac('sha1', salt + '').update(str + '').digest('hex');
};
/**
* Create HMAC sha512 hash from provided value and optional salt
* @param {string} str - value to hash
* @param {string=} addSalt - optional salt, uses ms timestamp by default
* @returns {string} HMAC sha1 hash
*/
common.sha512Hash = function(str, addSalt) {
var salt = (addSalt) ? new Date().getTime() : '';
return crypto.createHmac('sha512', salt + '').update(str + '').digest('hex');
};
/**
* Create argon2 hash string
* @param {string} str - string to hash
* @returns {promise} hash promise
**/
common.argon2Hash = function(str) {
return argon2.hash(str);
};
/**
* Create MD5 hash from provided value
* @param {string} str - value to hash
* @returns {string} MD5 hash
*/
common.md5Hash = function(str) {
return crypto.createHash('md5').update(str + '').digest('hex');
};
/**
* Modifies provided object in the format object["2012.7.20.property"] = increment.
* Usualy used when filling up Countly metric model data
* @param {params} params - {@link params} object
* @param {object} object - object to fill
* @param {string} property - meric value or segment or property to fill/increment
* @param {number=} increment - by how much to increments, default is 1
* @returns {void} void
* @example
* var obj = {};
* common.fillTimeObject(params, obj, "u", 1);
* console.log(obj);
* //outputs
* { '2017.u': 1,
* '2017.2.u': 1,
* '2017.2.23.u': 1,
* '2017.2.23.8.u': 1,
* '2017.w8.u': 1 }
*/
common.fillTimeObject = function(params, object, property, increment) {
increment = (increment) ? increment : 1;
var timeObj = params.time;
if (!timeObj || !timeObj.yearly || !timeObj.monthly || !timeObj.weekly || !timeObj.daily || !timeObj.hourly) {
return false;
}
object[timeObj.yearly + '.' + property] = increment;
object[timeObj.monthly + '.' + property] = increment;
object[timeObj.daily + '.' + property] = increment;
// If the property parameter contains a dot, hourly data is not saved in
// order to prevent two level data (such as 2012.7.20.TR.u) to get out of control.
if (property.indexOf('.') === -1) {
object[timeObj.hourly + '.' + property] = increment;
}
// For properties that hold the unique visitor count we store weekly data as well.
if (property.substr(-2) === ("." + common.dbMap.unique) ||
property === common.dbMap.unique ||
property.substr(0, 2) === (common.dbMap.frequency + ".") ||
property.substr(0, 2) === (common.dbMap.loyalty + ".") ||
property.substr(0, 3) === (common.dbMap.durations + ".") ||
property === common.dbMap.paying) {
object[timeObj.yearly + ".w" + timeObj.weekly + '.' + property] = increment;
}
};
/**
* Creates a time object from request's milisecond or second timestamp in provided app's timezone
* @param {string} appTimezone - app's timezone
* @param {number} reqTimestamp - timestamp in the request
* @returns {timeObject} Time object for current request
*/
common.initTimeObj = function(appTimezone, reqTimestamp) {
var currTimestamp,
curMsTimestamp,
tmpMoment,
currDateWithoutTimestamp = moment();
// Check if the timestamp parameter exists in the request and is a 10 or 13 digit integer, handling also float timestamps with ms after dot
if (reqTimestamp && (Math.round(parseFloat(reqTimestamp, 10)) + "").length === 10 && common.isNumber(reqTimestamp)) {
// If the received timestamp is greater than current time use the current time as timestamp
currTimestamp = (parseInt(reqTimestamp, 10) > currDateWithoutTimestamp.unix() + (60 * 60)) ? currDateWithoutTimestamp.unix() : parseInt(reqTimestamp, 10);
curMsTimestamp = (parseInt(reqTimestamp, 10) > currDateWithoutTimestamp.unix() + (60 * 60)) ? currDateWithoutTimestamp.valueOf() : parseFloat(reqTimestamp, 10) * 1000;
tmpMoment = moment(currTimestamp * 1000);
}
else if (reqTimestamp && (Math.round(parseFloat(reqTimestamp, 10)) + "").length === 13 && common.isNumber(reqTimestamp)) {
var tmpTimestamp = Math.floor(parseInt(reqTimestamp, 10) / 1000);
currTimestamp = (tmpTimestamp > currDateWithoutTimestamp.unix() + (60 * 60)) ? currDateWithoutTimestamp.unix() : tmpTimestamp;
curMsTimestamp = (tmpTimestamp > currDateWithoutTimestamp.unix() + (60 * 60)) ? currDateWithoutTimestamp.valueOf() : parseInt(reqTimestamp, 10);
tmpMoment = moment(currTimestamp * 1000);
}
else {
tmpMoment = moment();
currTimestamp = tmpMoment.unix(); // UTC
curMsTimestamp = tmpMoment.valueOf();
}
if (appTimezone) {
currDateWithoutTimestamp.tz(appTimezone);
tmpMoment.tz(appTimezone);
}
/**
* @typedef timeObject
* @type {object}
* @global
* @property {momentjs} now - momentjs instance for request's time in app's timezone
* @property {momentjs} nowUTC - momentjs instance for request's time in UTC
* @property {momentjs} nowWithoutTimestamp - momentjs instance for current time in app's timezone
* @property {number} timestamp - request's seconds timestamp
* @property {number} mstimestamp - request's miliseconds timestamp
* @property {string} yearly - year of request time in app's timezone in YYYY format
* @property {string} monthly - month of request time in app's timezone in YYYY.M format
* @property {string} daily - date of request time in app's timezone in YYYY.M.D format
* @property {string} hourly - hour of request time in app's timezone in YYYY.M.D.H format
* @property {number} weekly - week of request time in app's timezone as result day of the year, divided by 7
* @property {string} month - month of request time in app's timezone in format M
* @property {string} day - day of request time in app's timezone in format D
* @property {string} hour - hour of request time in app's timezone in format H
*/
return {
now: tmpMoment,
nowUTC: tmpMoment.clone().utc(),
nowWithoutTimestamp: currDateWithoutTimestamp,
timestamp: currTimestamp,
mstimestamp: curMsTimestamp,
yearly: tmpMoment.format("YYYY"),
monthly: tmpMoment.format("YYYY.M"),
daily: tmpMoment.format("YYYY.M.D"),
hourly: tmpMoment.format("YYYY.M.D.H"),
weekly: Math.ceil(tmpMoment.format("DDD") / 7),
month: tmpMoment.format("M"),
day: tmpMoment.format("D"),
hour: tmpMoment.format("H")
};
};
/**
* Creates a Date object from provided seconds timestamp in provided timezone
* @param {number} timestamp - unix timestamp in seconds
* @param {string} timezone - name of the timezone
* @returns {moment} moment object for provided time
*/
common.getDate = function(timestamp, timezone) {
var tmpDate = (timestamp) ? moment.unix(timestamp) : moment();
if (timezone) {
tmpDate.tz(timezone);
}
return tmpDate;
};
/**
* Returns day of the year from provided seconds timestamp in provided timezone
* @param {number} timestamp - unix timestamp in seconds
* @param {string} timezone - name of the timezone
* @returns {number} current day of the year
*/
common.getDOY = function(timestamp, timezone) {
var endDate = (timestamp) ? moment.unix(timestamp * 1000) : moment();
if (timezone) {
endDate.tz(timezone);
}
return endDate.dayOfYear();
};
/**
* Returns amount of days in provided year
* @param {number} year - year to check for days
* @returns {number} number of days in provided year
*/
common.getDaysInYear = function(year) {
if (new Date(year, 1, 29).getMonth() === 1) {
return 366;
}
else {
return 365;
}
};
/**
* Returns amount of iso weeks in provided year
* @param {number} year - year to check for days
* @returns {number} number of iso weeks in provided year
*/
common.getISOWeeksInYear = function(year) {
var d = new Date(year, 0, 1),
isLeap = new Date(year, 1, 29).getMonth() === 1;
//Check for a Jan 1 that's a Thursday or a leap year that has a
//Wednesday Jan 1. Otherwise it's 52
return d.getDay() === 4 || isLeap && d.getDay() === 3 ? 53 : 52;
};
/**
* Validates provided arguments
* @param {object} args - arguments to validate
* @param {object} argProperties - rules for validating each argument
* @param {boolean} argProperties.required - should property be present in args
* @param {string} argProperties.type - what type should property be, possible values: String, Array, Number, URL, Boolean, Object, Email
* @param {string} argProperties.max-length - property should not be longer than provided value
* @param {string} argProperties.min-length - property should not be shorter than provided value
* @param {string} argProperties.exclude-from-ret-obj - should property be present in returned validated args object
* @param {string} argProperties.has-number - should string property has any number in it
* @param {string} argProperties.has-char - should string property has any latin character in it
* @param {string} argProperties.has-upchar - should string property has any upper cased latin character in it
* @param {string} argProperties.has-special - should string property has any none latin character in it
* @param {boolean} returnErrors - return error details as array or only boolean result
* @returns {object} validated args in obj property, or false as result property if args do not pass validation and errors array
*/
common.validateArgs = function(args, argProperties, returnErrors) {
if (arguments.length === 2) {
returnErrors = false;
}
var returnObj;
if (returnErrors) {
returnObj = {
result: true,
errors: [],
obj: {}
};
}
else {
returnObj = {};
}
if (!args) {
if (returnErrors) {
returnObj.result = false;
returnObj.errors.push("Missing 'args' parameter");
delete returnObj.obj;
return returnObj;
}
else {
return false;
}
}
for (var arg in argProperties) {
var argState = true;
if (argProperties[arg].required) {
if (args[arg] === void 0) {
if (returnErrors) {
returnObj.errors.push("Missing " + arg + " argument");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (args[arg] !== void 0) {
if (argProperties[arg].type) {
if (argProperties[arg].type === 'Number' || argProperties[arg].type === 'String') {
if (toString.call(args[arg]) !== '[object ' + argProperties[arg].type + ']') {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else if (argProperties[arg].type === 'URL') {
if (toString.call(args[arg]) !== '[object String]') {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
else if (args[arg] && !/^([a-z]([a-z]|\d|\+|-|\.)*):(\/\/(((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:)*@)?((\[(|(v[\da-f]{1,}\.(([a-z]|\d|-|\.|_|~)|[!$&'()*+,;=]|:)+))\])|((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]))|(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=])*)(:\d*)?)(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)*)*|(\/((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)*)*)?)|((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)+(\/(([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)*)*)|((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)){0})(\?((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)|[\uE000-\uF8FF]|\/|\?)*)?(#((([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(%[\da-f]{2})|[!$&'()*+,;=]|:|@)|\/|\?)*)?$/i.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push("Invalid url string " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else if (argProperties[arg].type === 'Email') {
if (toString.call(args[arg]) !== '[object String]') {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
else if (args[arg] && !/[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/i.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push("Invalid url string " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else if (argProperties[arg].type === 'Boolean') {
if (!(args[arg] !== true || args[arg] !== false || toString.call(args[arg]) !== '[object Boolean]')) {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
}
}
else if (argProperties[arg].type === 'Array') {
if (!Array.isArray(args[arg])) {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else if (argProperties[arg].type === 'Object') {
if (toString.call(args[arg]) !== '[object ' + argProperties[arg].type + ']' && !(!argProperties[arg].required && args[arg] === null)) {
if (returnErrors) {
returnObj.errors.push("Invalid type for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else {
if (returnErrors) {
returnObj.errors.push("Invalid type declaration for " + arg);
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
else {
if (toString.call(args[arg]) !== '[object String]') {
if (returnErrors) {
returnObj.errors.push(arg + " should be string");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['max-length']) {
if (args[arg].length > argProperties[arg]['max-length']) {
if (returnErrors) {
returnObj.errors.push("Length of " + arg + " is greater than max length value");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['min-length']) {
if (args[arg].length < argProperties[arg]['min-length']) {
if (returnErrors) {
returnObj.errors.push("Length of " + arg + " is lower than min length value");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['has-number']) {
if (!/\d/.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push(arg + " should has number");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['has-char']) {
if (!/[A-Za-z]/.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push(arg + " should has char");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['has-upchar']) {
if (!/[A-Z]/.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push(arg + " should has upchar");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argProperties[arg]['has-special']) {
if (!/[^A-Za-z\d]/.test(args[arg])) {
if (returnErrors) {
returnObj.errors.push(arg + " should has special character");
returnObj.result = false;
argState = false;
}
else {
return false;
}
}
}
if (argState && returnErrors && !argProperties[arg]['exclude-from-ret-obj']) {
returnObj.obj[arg] = args[arg];
}
else if (!returnErrors && !argProperties[arg]['exclude-from-ret-obj']) {
returnObj[arg] = args[arg];
}
}
}
if (returnErrors && !returnObj.result) {
delete returnObj.obj;
return returnObj;
}
else {
return returnObj;
}
};
/**
* Fix event keys before storing in database by removing dots and $ from the string, removing other prefixes and limiting length
* @param {string} eventKey - key value to fix
* @returns {string|false} escaped key or false if not possible to use key at all
*/
common.fixEventKey = function(eventKey) {
var shortEventName = eventKey.replace(/system\.|\.\.|\$/g, "");
if (shortEventName.length >= 128) {
return false;
}
else {
return shortEventName;
}
};
/**
* Block {@link module:api/utils/common.returnMessage} and {@link module:api/utils/common.returnOutput} from ouputting anything
* @param {params} params - params object
*/
common.blockResponses = function(params) {
params.blockResponses = true;
};
/**
* Unblock/allow {@link module:api/utils/common.returnMessage} and {@link module:api/utils/common.returnOutput} ouputting anything
* @param {params} params - params object
*/
common.unblockResponses = function(params) {
params.blockResponses = false;
};
/**
* Custom API response handler callback
* @typedef APICallback
* @callback APICallback
* @type {function}
* @global
* @param {bool} error - true if there was problem processing request, and false if request was processed successfully
* @param {string} responseMessage - what API returns
* @param {object} headers - what API would have returned to HTTP request
* @param {number} returnCode - HTTP code, what API would have returned to HTTP request
* @param {params} params - request context that was passed to requestProcessor, modified during request processing
*/
/**
* Return raw headers and body
* @param {params} params - params object
* @param {number} returnCode - http code to use
* @param {string} body - raw data to output
* @param {object} heads - headers to add to the output
*/
common.returnRaw = function(params, returnCode, body, heads) {
params.response = {
code: returnCode,
body: body
};
if (params && params.APICallback && typeof params.APICallback === 'function') {
if (!params.blockResponses && (!params.res || !params.res.finished)) {
if (!params.res) {
params.res = {};
}
params.res.finished = true;
params.APICallback(returnCode !== 200, body, heads, returnCode, params);
}
return;
}
//set provided in configuration headers
var headers = {};
if (heads) {
for (var i in heads) {
headers[i] = heads[i];
}
}
if (params && params.res && params.res.writeHead && !params.blockResponses) {
if (!params.res.finished) {
params.res.writeHead(returnCode, headers);
if (body) {
params.res.write(body);
}
params.res.end();
}
else {
console.error("Output already closed, can't write more");
console.trace();
console.log(params);
}
}
};
/**
* Output message as request response with provided http code
* @param {params} params - params object
* @param {number} returnCode - http code to use
* @param {string} message - Message to output, will be encapsulated in JSON object under result property
* @param {object} heads - headers to add to the output
*/
common.returnMessage = function(params, returnCode, message, heads) {
params.response = {
code: returnCode,
body: JSON.stringify({result: message}, escape_html_entities)
};
if (params && params.APICallback && typeof params.APICallback === 'function') {
if (!params.blockResponses && (!params.res || !params.res.finished)) {
if (!params.res) {
params.res = {};
}
params.res.finished = true;
params.APICallback(returnCode !== 200, JSON.stringify({result: message}), heads, returnCode, params);
}
return;
}
//set provided in configuration headers
var headers = {
'Content-Type': 'application/json; charset=utf-8'
};
var add_headers = (plugins.getConfig("security").api_additional_headers || "").replace(/\r\n|\r|\n/g, "\n").split("\n");
var parts;
for (let i = 0; i < add_headers.length; i++) {
if (add_headers[i] && add_headers[i].length) {
parts = add_headers[i].split(/:(.+)?/);
if (parts.length === 3) {
headers[parts[0]] = parts[1];
}
}
}
if (heads) {
for (let i in heads) {
headers[i] = heads[i];
}
}
if (params && params.res && params.res.writeHead && !params.blockResponses) {
if (!params.res.finished) {
params.res.writeHead(returnCode, headers);
if (params.qstring.callback) {
params.res.write(params.qstring.callback + '(' + JSON.stringify({result: message}, escape_html_entities) + ')');
}
else {
params.res.write(JSON.stringify({result: message}, escape_html_entities));
}
params.res.end();
}
else {
console.error("Output already closed, can't write more");
console.trace();
console.log(params);
}
}
};
/**
* Output message as request response with provided http code
* @param {params} params - params object
* @param {output} output - object to stringify and output
* @param {string} noescape - prevent escaping HTML entities
* @param {object} heads - headers to add to the output
*/
common.returnOutput = function(params, output, noescape, heads) {
var escape = noescape ? undefined : function(k, v) {
return escape_html_entities(k, v, true);
};
params.response = {
code: 200,
body: JSON.stringify(output, escape)
};
if (params && params.APICallback && typeof params.APICallback === 'function') {
if (!params.blockResponses && (!params.res || !params.res.finished)) {
if (!params.res) {
params.res = {};
}
params.res.finished = true;
params.APICallback(false, output, heads, 200, params);
}
return;
}
//set provided in configuration headers
var headers = {
'Content-Type': 'application/json; charset=utf-8'
};
var add_headers = (plugins.getConfig("security").api_additional_headers || "").replace(/\r\n|\r|\n/g, "\n").split("\n");
var parts;
for (let i = 0; i < add_headers.length; i++) {
if (add_headers[i] && add_headers[i].length) {
parts = add_headers[i].split(/:(.+)?/);
if (parts.length === 3) {
headers[parts[0]] = parts[1];
}
}
}
if (heads) {
for (let i in heads) {
headers[i] = heads[i];
}
}
if (params && params.res && params.res.writeHead && !params.blockResponses) {
if (!params.res.finished) {
params.res.writeHead(200, headers);
if (params.qstring.callback) {
params.res.write(params.qstring.callback + '(' + JSON.stringify(output, escape) + ')');
}
else {
params.res.write(JSON.stringify(output, escape));
}
params.res.end();
}
else {
console.error("Output already closed, can't write more");
console.trace();
console.log(params);
}
}
};
var ipLogger = common.log('ip:api');
/**
* Get IP address from request object
* @param {req} req - nodejs request object
* @returns {string} ip address
*/
common.getIpAddress = function(req) {
var ipAddress = (req) ? req.headers['x-forwarded-for'] || req.headers['x-real-ip'] || req.connection.remoteAddress || req.socket.remoteAddress || (req.connection.socket ? req.connection.socket.remoteAddress : '') : "";
/* Since x-forwarded-for: client, proxy1, proxy2, proxy3 */
var ips = ipAddress.split(',');
//if ignoreProxies not setup, use outmost left ip address
if (!countlyConfig.ignoreProxies || !countlyConfig.ignoreProxies.length) {
ipLogger.d("From %s found ip %s", ipAddress, ips[0]);
return stripPort(ips[0]);
}
//search for the outmost right ip address ignoring provided proxies
var ip = "";
for (var i = ips.length - 1; i >= 0; i--) {
ips[i] = stripPort(ips[i]);
var masks = false;
if (countlyConfig.ignoreProxies && countlyConfig.ignoreProxies.length) {
masks = countlyConfig.ignoreProxies.some(function(elem) {
return ips[i].startsWith(elem);
});
}
if (ips[i] !== "127.0.0.1" && (!countlyConfig.ignoreProxies || !masks)) {
ip = ips[i];
break;
}
}
ipLogger.d("From %s found ip %s", ipAddress, ip);
return ip;
};
/**
* This function takes ipv4 or ipv6 with possible port, removes port information and returns plain ip address
* @param {string} ip - ip address to check for port and return plain ip
* @returns {string} plain ip address
*/
function stripPort(ip) {
var parts = (ip + "").split(".");
//check if ipv4
if (parts.length === 4) {
return ip.split(":")[0].trim();
}
else {
parts = (ip + "").split(":");
if (parts.length === 9) {
parts.pop();
}
if (parts.length === 8) {
ip = parts.join(":");
//remove enclosing [] for ipv6 if they are there
if (ip[0] === "[") {
ip = ip.substring(1);
}
if (ip[ip.length - 1] === "]") {
ip = ip.slice(0, -1);
}
}
}
return (ip + "").trim();
}
/**
* Modifies provided object filling properties used in zero documents in the format object["2012.7.20.property"] = increment.
* Usualy used when filling up Countly metric model zero document
* @param {params} params - {@link params} object
* @param {object} object - object to fill
* @param {string} property - meric value or segment or property to fill/increment
* @param {number=} increment - by how much to increments, default is 1
* @returns {void} void
* @example
* var obj = {};
* common.fillTimeObjectZero(params, obj, "u", 1);
* console.log(obj);
* //outputs
* { 'd.u': 1, 'd.2.u': 1, 'd.w8.u': 1 }
*/
common.fillTimeObjectZero = function(params, object, property, increment) {
var tmpIncrement = (increment) ? increment : 1,
timeObj = params.time;
if (!timeObj || !timeObj.yearly || !timeObj.month) {
return false;
}
if (property instanceof Array) {
for (var i = 0; i < property.length; i++) {
object['d.' + property[i]] = tmpIncrement;
object['d.' + timeObj.month + '.' + property[i]] = tmpIncrement;
// For properties that hold the unique visitor count we store weekly data as well.
if (property[i].substr(-2) === ("." + common.dbMap.unique) ||
property[i] === common.dbMap.unique ||
property[i].substr(0, 2) === (common.dbMap.frequency + ".") ||
property[i].substr(0, 2) === (common.dbMap.loyalty + ".") ||
property[i].substr(0, 3) === (common.dbMap.durations + ".") ||
property[i] === common.dbMap.paying) {
object['d.' + "w" + timeObj.weekly + '.' + property[i]] = tmpIncrement;
}
}
}
else {
object['d.' + property] = tmpIncrement;
object['d.' + timeObj.month + '.' + property] = tmpIncrement;
if (property.substr(-2) === ("." + common.dbMap.unique) ||
property === common.dbMap.unique ||
property.substr(0, 2) === (common.dbMap.frequency + ".") ||
property.substr(0, 2) === (common.dbMap.loyalty + ".") ||
property.substr(0, 3) === (common.dbMap.durations + ".") ||
property === common.dbMap.paying) {
object['d.' + "w" + timeObj.weekly + '.' + property] = tmpIncrement;
}
}
return true;
};
/**
* Modifies provided object filling properties used in monthly documents in the format object["2012.7.20.property"] = increment.
* Usualy used when filling up Countly metric model monthly document
* @param {params} params - {@link params} object
* @param {object} object - object to fill
* @param {string} property - meric value or segment or property to fill/increment
* @param {number=} increment - by how much to increments, default is 1
* @param {boolean=} forceHour - force recording hour information too, dfault is false
* @returns {void} void
* @example
* var obj = {};
* common.fillTimeObjectMonth(params, obj, "u", 1);
* console.log(obj);
* //outputs
* { 'd.23.u': 1, 'd.23.12.u': 1 }
*/
common.fillTimeObjectMonth = function(params, object, property, increment, forceHour) {
var tmpIncrement = (increment) ? increment : 1,
timeObj = params.time;
if (!timeObj || !timeObj.yearly || !timeObj.month || !timeObj.weekly || !timeObj.day || !timeObj.hour) {
return false;
}
if (property instanceof Array) {
for (var i = 0; i < property.length; i++) {
object['d.' + timeObj.day + '.' + property[i]] = tmpIncrement;
// If the property parameter contains a dot, hourly data is not saved in
// order to prevent two level data (such as 2012.7.20.TR.u) to get out of control.
if (forceHour || property[i].indexOf('.') === -1) {
object['d.' + timeObj.day + '.' + timeObj.hour + '.' + property[i]] = tmpIncrement;
}
}
}
else {
object['d.' + timeObj.day + '.' + property] = tmpIncrement;
if (forceHour || property.indexOf('.') === -1) {
object['d.' + timeObj.day + '.' + timeObj.hour + '.' + property] = tmpIncrement;
}
}
return true;
};
/**
* Record data in Countly standard metric model
* Can be used by plugins to record data, similar to sessions and users, with optional segments
* @param {params} params - {@link params} object
* @param {string} collection - name of the collections where to store data
* @param {string} id - id to prefix document ids, like app_id or segment id, etc
* @param {array} metrics - array of metrics to record, as ["u","t", "n"]
* @param {number=} value - value to increment all metrics for, default 1
* @param {object} segments - object with segments to record data, key segment name and value segment value
* @param {array} uniques - names of the metrics, which should be treated as unique, and stored in 0 docs and be estimated on output
* @param {number} lastTimestamp - timestamp in seconds to be used to determine if unique metrics it unique for specific period
* @example
* //recording attribution
* common.recordCustomMetric(params, "campaigndata", campaignId, ["clk", "aclk"], 1, {pl:"Android", brw:"Chrome"}, ["clk"], user["last_click"]);
*/
common.recordCustomMetric = function(params, collection, id, metrics, value, segments, uniques, lastTimestamp) {
value = value || 1;
var updateUsersZero = {},
updateUsersMonth = {},
tmpSet = {};
if (metrics) {
for (let i = 0; i < metrics.length; i++) {
recordMetric(params, metrics[i], {
segments: segments,
value: value,
unique: (uniques && uniques.indexOf(metrics[i]) !== -1) ? true : false,
lastTimestamp: lastTimestamp
},
tmpSet, updateUsersZero, updateUsersMonth);
}
}
var dbDateIds = common.getDateIds(params);
if (Object.keys(updateUsersZero).length || Object.keys(tmpSet).length) {
var update = {
$set: {
m: dbDateIds.zero,
a: params.app_id + ""
}
};
if (Object.keys(updateUsersZero).length) {
update.$inc = updateUsersZero;
}
if (Object.keys(tmpSet).length) {
update.$addToSet = {};
for (let i in tmpSet) {
update.$addToSet[i] = {$each: tmpSet[i]};
}
}
common.writeBatcher.add(collection, id + "_" + dbDateIds.zero, update);
}
if (Object.keys(updateUsersMonth).length) {
common.writeBatcher.add(collection, id + "_" + dbDateIds.month, {
$set: {
m: dbDateIds.month,
a: params.app_id + ""
},
'$inc': updateUsersMonth
});
}
};
/**
* Record data in Countly standard metric model
* Can be used by plugins to record data, similar to sessions and users, with optional segments
* @param {params} params - {@link params} object
* @param {object} props - object defining what to record
* @param {string} props.collection - name of the collections where to store data
* @param {string} props.id - id to prefix document ids, like app_id or segment id, etc
* @param {object} props.metrics - object defining metrics to record, using key as metric name and value object for segmentation, unique, etc
* @param {number=} props.metrics[].value - value to increment current metric for, default 1
* @param {object} props.metrics[].segments - object with segments to record data, key segment name and value segment value or array of segment values
* @param {boolean} props.metrics[].unique - if metric should be treated as unique, and stored in 0 docs and be estimated on output
* @param {number} props.metrics[].lastTimestamp - timestamp in seconds to be used to determine if unique metrics it unique for specific period
* @param {array} props.metrics[].hourlySegments - array of segments that should have hourly data too (by default hourly data not recorded for segments)
* @example
* //recording attribution
* common.recordCustomMetric(params, "campaigndata", campaignId, ["clk", "aclk"], 1, {pl:"Android", brw:"Chrome"}, ["clk"], user["last_click"]);
*/
common.recordMetric = function(params, props) {
var updateUsersZero = {},
updateUsersMonth = {},
tmpSet = {};
for (let i in props.metrics) {
props.metrics[i].value = props.metrics[i].value || 1;
recordMetric(params, i, props.metrics[i], tmpSet, updateUsersZero, updateUsersMonth);
}
var dbDateIds = common.getDateIds(params);
if (Object.keys(updateUsersZero).length || Object.keys(tmpSet).length) {
var update = {
$set: {
m: dbDateIds.zero,
a: params.app_id + ""
}
};
if (Object.keys(updateUsersZero).length) {
update.$inc = updateUsersZero;
}
if (Object.keys(tmpSet).length) {
update.$addToSet = {};
for (let i in tmpSet) {
update.$addToSet[i] = {$each: tmpSet[i]};
}
}
common.writeBatcher.add(props.collection, props.id + "_" + dbDateIds.zero, update);
}
if (Object.keys(updateUsersMonth).length) {
common.writeBatcher.add(props.collection, props.id + "_" + dbDateIds.month, {
$set: {
m: dbDateIds.month,
a: params.app_id + ""
},
'$inc': updateUsersMonth
});
}
};
/**
* Record specific metric
* @param {params} params - params object
* @param {string} metric - metric to record
* @param {object} props - properties of a metric defining how to record it
* @param {object} tmpSet - object with already set meta properties
* @param {object} updateUsersZero - object with already set update for zero docs
* @param {object} updateUsersMonth - object with already set update for months docs
**/
function recordMetric(params, metric, props, tmpSet, updateUsersZero, updateUsersMonth) {
var zeroObjUpdate = [],
monthObjUpdate = [];
if (props.unique) {
if (props.lastTimestamp) {
var currDate = common.getDate(params.time.timestamp, params.appTimezone),
lastDate = common.getDate(props.lastTimestamp, params.appTimezone),
secInMin = (60 * (currDate.minutes())) + currDate.seconds(),
secInHour = (60 * 60 * (currDate.hours())) + secInMin,
secInMonth = (60 * 60 * 24 * (currDate.date() - 1)) + secInHour,
secInYear = (60 * 60 * 24 * (common.getDOY(params.time.timestamp, params.appTimezone) - 1)) + secInHour;
if (props.lastTimestamp < (params.time.timestamp - secInMin)) {
updateUsersMonth['d.' + params.time.day + '.' + params.time.hour + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInHour)) {
updateUsersMonth['d.' + params.time.day + '.' + metric] = props.value;
}
if (lastDate.year() + "" === params.time.yearly + "" &&
Math.ceil(lastDate.format("DDD") / 7) < params.time.weekly) {
updateUsersZero["d.w" + params.time.weekly + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInMonth)) {
updateUsersZero['d.' + params.time.month + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInYear)) {
updateUsersZero['d.' + metric] = props.value;
}
}
else {
common.fillTimeObjectZero(params, updateUsersZero, metric, props.value);
common.fillTimeObjectMonth(params, updateUsersMonth, metric, props.value);
}
}
else {
//zeroObjUpdate.push(metric);
monthObjUpdate.push(metric);
}
if (props.segments) {
for (var j in props.segments) {
if (Array.isArray(props.segments[j])) {
for (var k = 0; k < props.segments[j].length; k++) {
recordSegmentMetric(params, metric, j, props.segments[j][k], props, tmpSet, updateUsersZero, updateUsersMonth, zeroObjUpdate, monthObjUpdate);
}
}
else if (props.segments[j]) {
recordSegmentMetric(params, metric, j, props.segments[j], props, tmpSet, updateUsersZero, updateUsersMonth, zeroObjUpdate, monthObjUpdate);
}
}
}
common.fillTimeObjectZero(params, updateUsersZero, zeroObjUpdate, props.value);
common.fillTimeObjectMonth(params, updateUsersMonth, monthObjUpdate, props.value);
}
/**
* Record specific metric segment
* @param {params} params - params object
* @param {string} metric - metric to record
* @param {string} name - name of the segment to record
* @param {string} val - value of the segment to record
* @param {object} props - properties of a metric defining how to record it
* @param {object} tmpSet - object with already set meta properties
* @param {object} updateUsersZero - object with already set update for zero docs
* @param {object} updateUsersMonth - object with already set update for months docs
* @param {array} zeroObjUpdate - segments to fill for for zero docs
* @param {array} monthObjUpdate - segments to fill for months docs
**/
function recordSegmentMetric(params, metric, name, val, props, tmpSet, updateUsersZero, updateUsersMonth, zeroObjUpdate, monthObjUpdate) {
var escapedMetricKey = name.replace(/^\$/, "").replace(/\./g, ":");
var escapedMetricVal = (val + "").replace(/^\$/, "").replace(/\./g, ":");
if (!tmpSet["meta." + escapedMetricKey]) {
tmpSet["meta." + escapedMetricKey] = [];
}
tmpSet["meta." + escapedMetricKey].push(escapedMetricVal);
var recordHourly = (props.hourlySegments && props.hourlySegments.indexOf(name) !== -1) ? true : false;
if (props.unique) {
if (props.lastTimestamp) {
var currDate = common.getDate(params.time.timestamp, params.appTimezone),
lastDate = common.getDate(props.lastTimestamp, params.appTimezone),
secInMin = (60 * (currDate.minutes())) + currDate.seconds(),
secInHour = (60 * 60 * (currDate.hours())) + secInMin,
secInMonth = (60 * 60 * 24 * (currDate.date() - 1)) + secInHour,
secInYear = (60 * 60 * 24 * (common.getDOY(params.time.timestamp, params.appTimezone) - 1)) + secInHour;
if (props.lastTimestamp < (params.time.timestamp - secInMin)) {
updateUsersMonth['d.' + params.time.day + '.' + params.time.hour + '.' + escapedMetricVal + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInHour)) {
updateUsersMonth['d.' + params.time.day + '.' + escapedMetricVal + '.' + metric] = props.value;
}
if (lastDate.year() + "" === params.time.yearly + "" &&
Math.ceil(lastDate.format("DDD") / 7) < params.time.weekly) {
updateUsersZero["d.w" + params.time.weekly + '.' + escapedMetricVal + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInMonth)) {
updateUsersZero['d.' + params.time.month + '.' + escapedMetricVal + '.' + metric] = props.value;
}
if (props.lastTimestamp < (params.time.timestamp - secInYear)) {
updateUsersZero['d.' + escapedMetricVal + '.' + metric] = props.value;
}
}
else {
common.fillTimeObjectZero(params, updateUsersZero, escapedMetricVal + '.' + metric, props.value);
common.fillTimeObjectMonth(params, updateUsersMonth, escapedMetricVal + '.' + metric, props.value, recordHourly);
}
}
else {
if (recordHourly) {
//common.fillTimeObjectZero(params, updateUsersZero, escapedMetricVal + '.' + metric, props.value);
common.fillTimeObjectMonth(params, updateUsersMonth, escapedMetricVal + '.' + metric, props.value, recordHourly);
}
else {
//zeroObjUpdate.push(escapedMetricVal + "." + metric);
monthObjUpdate.push(escapedMetricVal + "." + metric);
}
}
}
/**
* Get object of date ids that should be used in fetching standard metric model documents
* @param {params} params - {@link params} object
* @returns {object} with date ids, as {zero:"2017:0", month:"2017:2"}
*/
common.getDateIds = function(params) {
if (!params || !params.time) {
return {
zero: "0000:0",
month: "0000:1"
};
}
return {
zero: params.time.yearly + ":0",
month: params.time.yearly + ":" + params.time.month
};
};
/**
* Get diference between 2 momentjs instances in specific measurement
* @param {moment} moment1 - momentjs with start date
* @param {moment} moment2 - momentjs with end date
* @param {string} measure - units of difference, can be minutes, hours, days, weeks
* @returns {number} difference in provided units
*/
common.getDiff = function(moment1, moment2, measure) {
var divider = 1;
switch (measure) {
case "minutes":
divider = 60;
break;
case "hours":
divider = 60 * 60;
break;
case "days":
divider = 60 * 60 * 24;
break;
case "weeks":
divider = 60 * 60 * 24 * 7;
break;
}
return Math.floor((moment1.unix() - moment2.unix()) / divider);
};
/**
* Compares two version strings with : as delimiter (which we used to escape dots in app versions)
* @param {string} v1 - first version
* @param {string} v2 - second version
* @param {object} options - providing additional options
* @param {string} options.delimiter - delimiter between version, subversion, etc, defaults :
* @param {string} options.zeroExtend - changes the result if one version string has less parts than the other. In this case the shorter string will be padded with "zero" parts instead of being considered smaller.
* @param {string} options.lexicographical - compares each part of the version strings lexicographically instead of naturally; this allows suffixes such as "b" or "dev" but will cause "1.10" to be considered smaller than "1.2".
* @returns {number} 0 if they are both the same, 1 if first one is higher and -1 is second one is higher
*/
common.versionCompare = function(v1, v2, options) {
var lexicographical = options && options.lexicographical,
zeroExtend = options && options.zeroExtend,
delimiter = options && options.delimiter || ":",
v1parts = v1.split(delimiter),
v2parts = v2.split(delimiter);
/**
* Check if provided version is correct
* @param {string} x - version to test
* @returns {boolean} if version is correct
**/
function isValidPart(x) {
return (lexicographical ? /^\d+[A-Za-z]*$/ : /^\d+$/).test(x);
}
if (!v1parts.every(isValidPart) || !v2parts.every(isValidPart)) {
return NaN;
}
if (zeroExtend) {
while (v1parts.length < v2parts.length) {
v1parts.push("0");
}
while (v2parts.length < v1parts.length) {
v2parts.push("0");
}
}
if (!lexicographical) {
v1parts = v1parts.map(Number);
v2parts = v2parts.map(Number);
}
for (var i = 0; i < v1parts.length; ++i) {
if (v2parts.length === i) {
return 1;
}
if (v1parts[i] === v2parts[i]) {
continue;
}
else if (v1parts[i] > v2parts[i]) {
return 1;
}
else {
return -1;
}
}
if (v1parts.length !== v2parts.length) {
return -1;
}
return 0;
};
/**
* Adjust timestamp with app's timezone for timestamp queries that should equal bucket results
* @param {number} ts - miliseconds timestamp
* @param {string} tz - timezone
* @returns {number} adjusted timestamp for timezone
*/
common.adjustTimestampByTimezone = function(ts, tz) {
var d = moment();
if (tz) {
d.tz(tz);
}
return ts + (d.utcOffset() * 60);
};
/**
* Getter/setter for dot notatons:
* @param {object} obj - object to use
* @param {string} is - path of properties to get
* @param {varies} value - value to set
* @returns {varies} value at provided path
* @example
* common.dot({a: {b: {c: 'string'}}}, 'a.b.c') === 'string'
* common.dot({a: {b: {c: 'string'}}}, ['a', 'b', 'c']) === 'string'
* common.dot({a: {b: {c: 'string'}}}, 'a.b.c', 5) === 5
* common.dot({a: {b: {c: 'string'}}}, 'a.b.c') === 5
*/
common.dot = function(obj, is, value) {
if (typeof is === 'string') {
return common.dot(obj, is.split('.'), value);
}
else if (is.length === 1 && value !== undefined) {
obj[is[0]] = value;
return value;
}
else if (is.length === 0) {
return obj;
}
else if (!obj) {
return obj;
}
else {
return common.dot(obj[is[0]], is.slice(1), value);
}
};
/**
* Not deep object and primitive type comparison function
*
* @param {Any} a object to compare
* @param {Any} b object to compare
* @param {Boolean} checkFromA true if check should be performed agains keys of a, resulting in true even if b has more keys
* @return {Boolean} true if objects are equal, false if different types or not equal
*/
common.equal = function(a, b, checkFromA) {
if (a === b) {
return true;
}
else if (typeof a !== typeof b) {
return false;
}
else if ((a === null && b !== null) || (a !== null && b === null)) {
return false;
}
else if ((a === undefined && b !== undefined) || (a !== undefined && b === undefined)) {
return false;
}
else if (typeof a === 'object') {
if (!checkFromA && Object.keys(a).length !== Object.keys(b).length) {
return false;
}
for (let k in a) {
if (a[k] !== b[k]) {
return false;
}
}
return true;
}
else {
return false;
}
};
/**
* Returns plain object with key set to value
* @param {varies} arguments - every odd value will be used as key and every event value as value for odd key
* @returns {object} new object with set key/value properties
*/
common.o = function() {
var o = {};
for (var i = 0; i < arguments.length; i += 2) {
o[arguments[i]] = arguments[i + 1];
}
return o;
};
/**
* Return index of array with objects where property = value
* @param {array} array - array where to search value
* @param {string} property - property where to look for value
* @param {varies} value - value you are searching for
* @returns {number} index of the array
*/
common.indexOf = function(array, property, value) {
for (var i = 0; i < array.length; i += 1) {
if (array[i][property] === value) {
return i;
}
}
return -1;
};
/**
* Optionally load module if it exists
* @param {string} module - module name
* @param {object} options - additional opeitons
* @param {boolean} options.rethrow - throw exception if there is some other error
* @param {varies} value - value you are searching for
* @returns {number} index of the array
*/
common.optional = function(module, options) {
try {
if (module[0] in {'.': 1}) {
module = process.cwd() + module.substr(1);
}
return require(module);
}
catch (err) {
if (err.code !== 'MODULE_NOT_FOUND' && options && options.rethrow) {
throw err;
}
}
return null;
};
/**
* Create promise for function which result should be checked periodically
* @param {function} func - function to run when verifying result, should return true if success
* @param {number} count - how many times to run the func before giving up, if result is always negative
* @param {number} interval - how often to retest function on negative result in miliseconds
* @returns {Promise} promise for checking task
*/
common.checkPromise = function(func, count, interval) {
return new Promise((resolve, reject) => {
/**
* Check promise
**/
function check() {
if (func()) {
resolve();
}
else if (count <= 0) {
reject('Timed out');
}
else {
count--;
setTimeout(check, interval);
}
}
check();
});
};
common.clearClashingQueryOperations = function(query) {
var map = {};
var field;
for (var opp in query) {
for (field in query[opp]) {
map[field] = (map[field] || 0) + 1;
}
}
var badPaths = [];
var allPaths = Object.keys(map);
for (var z = 0; z < allPaths.length; z++) {
for (var p = z + 1; p < allPaths.length; p++) {
if (allPaths[z].startsWith(allPaths[p] + ".")) {
map[allPaths[z]]++;
map[allPaths[p]]++;
}
}
}
for (var path in map) {
if (map[path] > 1) {
badPaths.push(path);
}
}
if (badPaths.length > 0) {
var droppedOp = [];
var st = JSON.stringify(query);
for (var op in query) {
for (field in query[op]) {
if (badPaths.indexOf(field) > -1) {
droppedOp.push("{" + op + ":{" + field + ":" + JSON.stringify(query[op][field]) + "}}");
delete query[op][field];
if (Object.keys(query[op]).length === 0) {
delete query[op];
}
}
}
}
console.log("Conflicting operations. Query:" + st + " OPS:" + droppedOp.join(",") + " Resulted query:" + JSON.stringify(query));
}
return query;
};
/**
* Single method to update app_users document for specific user for SDK requests
* @param {params} params - params object
* @param {object} update - update query for mongodb, should contain operators on highest level, as $set or $unset
* @param {boolean} no_meta - if true, won't update some auto meta data, like first api call, last api call, etc.
* @param {function} callback - function to run when update is done or failes, passing error and result as arguments
*/
common.updateAppUser = function(params, update, no_meta, callback) {
//backwards compatability
if (typeof no_meta === "function") {
callback = no_meta;
no_meta = false;
}
if (Object.keys(update).length) {
for (var i in update) {
if (i.indexOf("$") !== 0) {
let err = "Unkown modifier " + i + " in " + update + " for " + params.href;
console.log(err);
if (callback) {
callback(err);
}
return;
}
}
var user = params.app_user || {};
if (!params.qstring.device_id && typeof user.did === "undefined") {
let err = "Device id is not provided for" + params.href;
console.log(err);
if (callback) {
callback(err);
}
return;
}
if (!no_meta && !params.qstring.no_meta) {
if (typeof user.fac === "undefined") {
if (!update.$set) {
update.$set = {};
}
if (!update.$set.fac) {
if (user.fs && user.fs < params.time.timestamp) {
update.$set.fac = user.fs;
}
else {
update.$set.fac = params.time.timestamp;
}
}
update.$set.first_sync = Math.round(Date.now() / 1000);
}
if (typeof user.lac === "undefined" || (user.lac + "").length === 13 || user.lac < params.time.timestamp) {
if (!update.$set) {
update.$set = {};
}
if (!update.$set.lac) {
update.$set.lac = params.time.timestamp;
}
update.$set.last_sync = Math.round(Date.now() / 1000);
}
if (!user.sdk) {
user.sdk = {};
}
if (params.qstring.sdk_name && params.qstring.sdk_name !== user.sdk.name) {
if (!update.$set) {
update.$set = {};
}
update.$set["sdk.name"] = params.qstring.sdk_name;
}
if (params.qstring.sdk_version && params.qstring.sdk_version !== user.sdk.version) {
if (!update.$set) {
update.$set = {};
}
update.$set["sdk.version"] = params.qstring.sdk_version;
}
if (plugins.getConfig("api", params.app && params.app.plugins, true).prevent_duplicate_requests && user.last_req !== params.request_hash) {
if (!update.$set) {
update.$set = {};
}
update.$set.last_req = params.request_hash;
}
}
if (params.qstring.device_id && typeof user.did === "undefined") {
if (!update.$set) {
update.$set = {};
}
if (!update.$set.did) {
update.$set.did = params.qstring.device_id;
}
}
if (callback) {
common.db.collection('app_users' + params.app_id).findAndModify({'_id': params.app_user_id}, {}, common.clearClashingQueryOperations(update), {
new: true,
upsert: true
}, function(err, res) {
if (!err && res && res.value) {
params.app_user = res.value;
}
callback(err, res);
});
}
else {
// using updateOne costs less than findAndModify, so we should use this
// when acknowledging writes and updated information is not relevant (aka callback is not passed)
common.db.collection('app_users' + params.app_id).updateOne({'_id': params.app_user_id}, common.clearClashingQueryOperations(update), {upsert: true}, function() {});
}
}
else if (callback) {
callback();
}
};
/**
* Update carrier from metrics to convert mnc/mcc code to carrier name
* @param {object} metrics - metrics object from SDK request
*/
common.processCarrier = function(metrics) {
if (metrics && metrics._carrier) {
var carrier = metrics._carrier + "";
//random hash without spaces
if (carrier.length === 16 && carrier.indexOf(" ") === -1) {
delete metrics._carrier;
return;
}
//random code
if ((carrier.length === 5 || carrier.length === 6) && /^[0-9]+$/.test(carrier)) {
//check if mcc and mnc match some operator
var arr = mcc_mnc_list.filter({ mccmnc: carrier });
if (arr && arr.length && (arr[0].brand || arr[0].operator)) {
carrier = arr[0].brand || arr[0].operator;
}
else {
delete metrics._carrier;
return;
}
}
carrier = carrier.replace(/\w\S*/g, function(txt) {
return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
});
metrics._carrier = carrier;
}
};
/**
* Parse Sequence
* @param {number} num - sequence number for id
* @returns {string} converted to base 62 number
*/
common.parseSequence = (num) => {
const valSeq = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M",
"N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"];
const digits = [];
const base = valSeq.length;
let result = "";
while (num > base - 1) {
digits.push(num % base);
num = Math.floor(num / base);
}
digits.push(num);
for (let i = digits.length - 1; i >= 0; --i) {
result = result + valSeq[digits[i]];
}
return result;
};
/**
* Promise that tries to catch errors
* @param {function} f function which is usually passed to Promise constructor
* @return {Promise} Promise with constructor catching errors by rejecting the promise
*/
common.p = f => {
return new Promise((res, rej) => {
try {
f(res, rej);
}
catch (e) {
rej(e);
}
});
};
/**
* Revive json encoded data, as for example, regular expressions
* @param {string} key - key of json object
* @param {vary} value - value of json object
* @returns {vary} modified value, if it had revivable data
*/
common.reviver = (key, value) => {
if (value.toString().indexOf("__REGEXP ") === 0) {
const m = value.split("__REGEXP ")[1].match(/\/(.*)\/(.*)?/);
return new RegExp(m[1], m[2] || "");
}
else {
return value;
}
};
/**
* Generate random password
* @param {number} length - length of the password
* @param {boolean} no_special - do not include special characters
* @returns {string} password
* @example
* //outputs 4UBHvRBG1v
* common.generatePassword(10, true);
*/
common.generatePassword = function(length, no_special) {
var text = [];
var chars = "abcdefghijklmnopqrstuvwxyz";
var upchars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
var numbers = "0123456789";
var specials = '!@#$%^&*()_+{}:"<>?|[];\',./`~';
var all = chars + upchars + numbers;
if (!no_special) {
all += specials;
}
//1 char
text.push(upchars.charAt(Math.floor(Math.random() * upchars.length)));
//1 number
text.push(numbers.charAt(Math.floor(Math.random() * numbers.length)));
//1 special char
if (!no_special) {
text.push(specials.charAt(Math.floor(Math.random() * specials.length)));
length--;
}
var j, x, i;
//5 any chars
for (i = 0; i < Math.max(length - 2, 5); i++) {
text.push(all.charAt(Math.floor(Math.random() * all.length)));
}
//randomize order
for (i = text.length; i; i--) {
j = Math.floor(Math.random() * i);
x = text[i - 1];
text[i - 1] = text[j];
text[j] = x;
}
return text.join("");
};
/**
* Check db host match for both of API and Frontend config
* @param {object} apiConfig - mongodb object from API config
* @param {object} frontendConfig - mongodb object from Frontend config
* @returns {boolean} isMatched - is config correct?
*/
common.checkDatabaseConfigMatch = (apiConfig, frontendConfig) => {
if (typeof apiConfig === typeof frontendConfig) {
if (typeof apiConfig === "string") {
// mongodb://mongodb0.example.com:27017/admin
if (!apiConfig.includes("@") && !frontendConfig.includes("@")) {
// mongodb0.example.com:27017
if (apiConfig.includes('/') && frontendConfig.includes('/')) {
try {
let apiMongoHost = apiConfig.split("/")[2];
let frontendMongoHost = frontendConfig.split("/")[2];
let apiMongoDb,
frontendMongoDb;
if (apiConfig.includes('?')) {
apiMongoDb = apiConfig.split("/")[3].split('?')[0];
}
else {
apiMongoDb = apiConfig.split("/")[3];
}
if (frontendConfig.includes('?')) {
frontendMongoDb = frontendConfig.split("/")[3].split('?')[0];
}
else {
frontendMongoDb = frontendConfig.split("/")[3];
}
if (apiMongoHost === frontendMongoHost && apiMongoDb === frontendMongoDb) {
return true;
}
else {
return false;
}
}
catch (splitErrorBasicString) {
return false;
}
}
else {
return false;
}
}
//mongodb://myDBReader:D1fficultP%40ssw0rd@mongodb0.example.com:27017/admin
else if (apiConfig.includes("@") && frontendConfig.includes("@")) {
if (apiConfig.includes('/') && frontendConfig.includes('/')) {
try {
let apiMongoHost = apiConfig.split("@")[1].split("/")[0];
let apiMongoDb,
frontendMongoDb;
if (apiConfig.includes('?')) {
apiMongoDb = apiConfig.split("@")[1].split("/")[1].split('?')[0];
}
else {
apiMongoDb = apiConfig.split("@")[1].split("/")[1];
}
let frontendMongoHost = frontendConfig.split("@")[1].split("/")[0];
if (frontendConfig.includes('?')) {
frontendMongoDb = frontendConfig.split("@")[1].split("/")[1].split('?')[0];
}
else {
frontendMongoDb = frontendConfig.split("@")[1].split("/")[1];
}
if (apiMongoHost === frontendMongoHost && apiMongoDb === frontendMongoDb) {
return true;
}
else {
return false;
}
}
catch (splitErrorComplexString) {
return false;
}
}
else {
return false;
}
}
else {
return false;
}
}
else if (typeof apiConfig === "object") {
/**
* {
* mongodb: {
* host: 'localhost',
*
* }
* }
*/
if (Object.prototype.hasOwnProperty.call(apiConfig, 'host') && Object.prototype.hasOwnProperty.call(frontendConfig, 'host')) {
if (apiConfig.host === frontendConfig.host && apiConfig.db === frontendConfig.db) {
return true;
}
else {
return false;
}
}
/**
* {
* mongodb: {
* replSetServers: [
* '192.168.3.1:27017',
* '192.168.3.2:27017
* ]
* }
* }
*/
else if (Object.prototype.hasOwnProperty.call(apiConfig, 'replSetServers') && Object.prototype.hasOwnProperty.call(frontendConfig, 'replSetServers')) {
if (apiConfig.replSetServers.length === frontendConfig.replSetServers.length && apiConfig.db === frontendConfig.db) {
let isCorrect = true;
for (let i = 0; i < apiConfig.replSetServers.length; i++) {
if (apiConfig.replSetServers[i] !== frontendConfig.replSetServers[i]) {
isCorrect = false;
}
}
return isCorrect;
}
else {
return false;
}
}
else {
return false;
}
}
else {
return false;
}
}
else {
return false;
}
};
/**
* Sanitizes a filename to prevent directory traversals and such.
* @param {string} filename - filename to sanitize
* @param {string} replacement - string to replace characters to be removed
* @returns {string} sanitizedFilename - sanitized filename
*/
common.sanitizeFilename = (filename, replacement = "") => {
return (filename + "")
.replace(/[\x00-\x1f\x80-\x9f]+/g, replacement)
.replace(/[\/\?<>\\:\*\|"]/g, replacement)
.replace(/^\.{1,2}$/, replacement)
.replace(/^\.+/, replacement);
};
/**
* Merge 2 mongodb update queries
* @param {object} ob1 - existing database update query
* @param {object} ob2 - addition to database update query
* @returns {object} merged database update query
*/
common.mergeQuery = function(ob1, ob2) {
if (ob2) {
for (let key in ob2) {
if (!ob1[key]) {
ob1[key] = ob2[key];
}
else if (key === "$set" || key === "$setOnInsert" || key === "$unset") {
for (let val in ob2[key]) {
ob1[key][val] = ob2[key][val];
}
}
else if (key === "$addToSet") {
for (let val in ob2[key]) {
if (typeof ob1[key][val] !== 'object') {
ob1[key][val] = {'$each': [ob1[key][val]]}; //create as object if it is single value
}
if (typeof ob2[key][val] === 'object' && ob2[key][val].$each) {
for (let p = 0; p < ob2[key][val].$each.length; p++) {
if (ob1[key][val].$each.indexOf(ob2[key][val].$each[p]) === -1) {
ob1[key][val].$each.push(ob2[key][val].$each[p]);
}
}
}
else {
if (ob1[key][val].$each.indexOf(ob2[key][val]) === -1) {
ob1[key][val].$each.push(ob2[key][val]);
}
}
}
}
else if (key === "$push") {
for (let val in ob2[key]) {
if (typeof ob1[key][val] !== 'object') {
ob1[key][val] = {'$each': [ob1[key][val]]};
}
if (typeof ob2[key][val] === 'object' && ob2[key][val].$each) {
for (let p = 0; p < ob2[key][val].$each.length; p++) {
ob1[key][val].$each.push(ob2[key][val].$each[p]);
}
//copy other push modifiers
for (let modifier in ob2[key][val]) {
if (modifier !== "$each") {
ob1[key][val][modifier] = ob2[key][val][modifier];
}
}
}
else {
ob1[key][val].$each.push(ob2[key][val]);
}
}
}
else if (key === "$inc") {
for (let val in ob2[key]) {
ob1[key][val] = ob1[key][val] || 0;
ob1[key][val] += ob2[key][val];
}
}
else if (key === "$mul") {
for (let val in ob2[key]) {
ob1[key][val] = ob1[key][val] || 0;
ob1[key][val] *= ob2[key][val];
}
}
else if (key === "$min") {
for (let val in ob2[key]) {
ob1[key][val] = ob1[key][val] || ob2[key][val];
ob1[key][val] = Math.min(ob1[key][val], ob2[key][val]);
}
}
else if (key === "$max") {
for (let val in ob2[key]) {
ob1[key][val] = ob1[key][val] || ob2[key][val];
ob1[key][val] = Math.max(ob1[key][val], ob2[key][val]);
}
}
}
//try to fix colliding fields
if (ob1 && ob1.$set && ob1.$set.data && ob1.$inc) {
for (let key in ob1.$inc) {
if (key.startsWith("data.")) {
ob1.$set.data[key.replace("data.", "")] = ob1.$inc[key];
delete ob1.$inc[key];
}
}
}
}
return ob1;
};
/**
* DataTable is a helper class for data tables in the UI which have bServerSide: true. It provides
* abstraction for server side pagination, searching and column based sorting. The class relies
* on MongoDB's aggregation for all operations. This doesn't include making db calls though. Since
* there can be many different execution scenarios, db left to the users of the class.
*
* There are two main methods of the class:
*
* 1) getAggregationPipeline: Creates a pipeline which can be executed by MongoDB. The pipeline
* can be customized, please see its description.
*
* 2) getProcessedResult: Processes the aggregation result. Returns an object, which is ready to be
* served as a response directly.
*/
class DataTable {
/**
* Constructor
* @param {Object} queryString This object should contain the datatable arguments like iDisplayStart,
* iDisplayEnd, etc. These are added to request by DataTables automatically. If you have a different
* use-case, please make sure that the object has necessary fields.
* @param {('full'|'rows')} queryString.outputFormat The default output of getProcessedResult is a
* DataTable compatible object ("full"). However, some consumers of the API may require simple, array-like
* results too ("rows"). In order to allow consumers to specify expected output, the field can be used.
* @param {Object} options Wraps options
* @param {Array} options.columnOrder If there are sortable columns in the table, then you need to
* specify a column list in order to make it work (e.g. ["name", "status"]).
* @param {Object} options.defaultSorting When there is no sorting provided in query string, sorting
* falls back to this object, if you provide any (e.g. {"name": "asc"}).
* @param {Array} options.searchableFields Specify searchable fields of a record/item (e.g. ["name", "description"]).
* @param {('regex'|'hard')} options.searchStrategy Specify searching method. If "regex", then a regex
* search is performed on searchableFields. Other values will be considered as hard match.
* @param {Object} options.outputProjection Adds a $project stage to the output rows using the object passed.
* @param {('full'|'rows')} options.defaultOutputFormat This is the default value for queryString.outputFormat.
* @param {String} options.uniqueKey A generic-purpose unique key for records. Default is _id, as it
* is the default identifier of MongoDB docs. Please make sure that this key is in the output of initial pipeline.
* @param {Boolean} options.disableUniqueSorting When sorting is done, the uniqueKey is automatically
* injected to the sorting expression, in order to mitigate possible duplicate records in pages. This is
* a protection for cases when the sorting is done based on non-unique fields. Injection is enabled by default.
* If you want to disable this feature, pass true.
*/
constructor(queryString, {
columnOrder = [],
defaultSorting = null,
searchableFields = [],
searchStrategy = "regex",
outputProjection = null,
defaultOutputFormat = "full",
uniqueKey = "_id",
disableUniqueSorting = false
} = {}) {
this.queryString = queryString;
this.skip = null;
this.limit = null;
this.searchTerm = null;
this.sorting = null;
this.echo = "0";
//
this.columnOrder = columnOrder;
this.defaultSorting = defaultSorting;
this.searchableFields = searchableFields;
this.searchStrategy = searchStrategy;
this.outputProjection = outputProjection;
this.defaultOutputFormat = defaultOutputFormat;
this.uniqueKey = uniqueKey;
this.disableUniqueSorting = disableUniqueSorting;
//
if (this.columnOrder && this.columnOrder.length > 0) {
if (this.queryString.iSortCol_0 && this.queryString.sSortDir_0) {
var sortField = this.columnOrder[parseInt(this.queryString.iSortCol_0, 10)];
if (sortField) {
this.sorting = {[sortField]: this.queryString.sSortDir_0};
}
}
}
if (!this.sorting && this.defaultSorting) {
this.sorting = this.defaultSorting;
}
if (this.sorting) {
var _tempSorting = {};
for (var sortKey in this.sorting) {
if (this.sorting[sortKey] === "asc") {
_tempSorting[sortKey] = 1;
}
else {
_tempSorting[sortKey] = -1;
}
}
if (this.disableUniqueSorting !== true && !_tempSorting[this.uniqueKey]) {
_tempSorting[this.uniqueKey] = 1;
}
this.sorting = _tempSorting;
}
if (this.queryString.iDisplayStart) {
this.skip = parseInt(this.queryString.iDisplayStart, 10);
}
if (this.queryString.iDisplayLength) {
this.limit = parseInt(this.queryString.iDisplayLength, 10);
}
if (this.queryString.sSearch && this.queryString.sSearch !== "") {
this.searchTerm = this.queryString.sSearch;
}
if (this.queryString.sEcho) {
this.echo = this.queryString.sEcho;
}
}
/**
* Returns the search field for. Only for internal use.
* @returns {Object|String} Regex object or search term itself
*/
_getSearchField() {
if (this.searchStrategy === "regex") {
return {$regex: this.searchTerm, $options: 'i'};
}
return this.searchTerm;
}
/**
* Creates an aggregation pipeline based on the query string and additional stages/facets
* if provided any. Data flow between stages are not checked, so please do check manually.
*
* @param {Object} options Wraps options
* @param {Array} options.initialPipeline If you need to select a subset, to add new fields or
* anything else involving aggregation stages, you can pass an array of stages using options.initialPipeline.
* Initial pipeline is basically used for counting the total number of documents without pagination and search.
*
* # of output rows = total number of docs.
*
* @param {Array} options.filteredPipeline Filtered pipeline will contain the remaining rows tested against a
* search query (if any). That is, this pipeline will get only the filtered docs as its input. If there is no
* query, then this will be another stage after initialPipeline. Paging and sorting are added after filteredPipeline.
*
* # of output rows = filtered number of docs.
*
* @param {Object} options.customFacets You can add facets to your results using option.customFacets.
* Custom facets will use initial pipeline's output as its input. If the documents you're
* looking for are included by initial pipeline's output, you can use this to avoid extra db calls.
* You can obtain outputs of your custom facets via getProcessedResult. Please note that custom facets will only be
* available when the output format is "full".
*
* @returns {Object} Pipeline object
*/
getAggregationPipeline({
initialPipeline = [],
filteredPipeline = [],
customFacets = {}
} = {}) {
var pipeline = [...initialPipeline]; // Initial pipeline (beforeMatch)
var $facetPagedData = [];
var $facetFilteredTotal = [];
if (this.searchTerm !== null && this.searchableFields && this.searchableFields.length > 0) {
var matcher = null;
if (this.searchableFields.length === 1) {
matcher = { [this.searchableFields[0]]: this._getSearchField() };
}
else {
var searchOr = [];
this.searchableFields.forEach((field) => {
searchOr.push({ [field]: this._getSearchField() });
});
matcher = { $or: searchOr};
}
$facetPagedData.push({$match: matcher});
$facetFilteredTotal.push({$match: matcher});
}
$facetPagedData.push(...filteredPipeline);
$facetFilteredTotal.push(...filteredPipeline); // TODO: optimize (no need to do pipeline operations unless there is match)
$facetFilteredTotal.push({$group: {"_id": null, "value": {$sum: 1}}});
if (this.sorting !== null) {
$facetPagedData.push({$sort: this.sorting});
}
if (this.skip !== null) {
$facetPagedData.push({$skip: this.skip});
}
if (this.limit !== null && this.limit > 0) {
$facetPagedData.push({$limit: this.limit});
}
if (this.outputProjection !== null) {
$facetPagedData.push({$project: this.outputProjection});
}
pipeline.push({
$facet:
{
...customFacets,
fullTotal: [{$group: {"_id": null, "value": {$sum: 1}}}],
filteredTotal: $facetFilteredTotal,
pagedData: $facetPagedData,
}
});
return pipeline;
}
/**
* Processes the aggregation result and returns a ready-to-use response.
* @param {Object} queryResult Aggregation result returned by the MongoDB.
* @param {Function} processFn A callback function that has a single argument 'rows'.
* As the name implies, it is an array of returned rows. The function can be used as
* a final stage to do modifications to fetched items before completing the response.
* @returns {Object|Array} Returns the final response
*/
getProcessedResult(queryResult, processFn) {
var fullTotal = 0,
filteredTotal = 0,
pagedData = [];
var customFacetResults = {};
if (queryResult && queryResult[0]) {
var facets = queryResult[0];
if (facets.fullTotal && facets.fullTotal[0] && facets.fullTotal[0].value) {
fullTotal = facets.fullTotal[0].value;
}
if (facets.filteredTotal && facets.filteredTotal[0] && facets.filteredTotal[0].value) {
filteredTotal = facets.filteredTotal[0].value;
}
if (facets.pagedData) {
pagedData = facets.pagedData;
}
for (var key in facets) {
if (["fullTotal", "filteredTotal", "pagedData"].includes(key)) {
continue;
}
customFacetResults[key] = facets[key];
}
}
if (processFn) {
var processed = processFn(pagedData);
if (processed) {
pagedData = processed;
}
}
var outputFormat = this.queryString.outputFormat || this.defaultOutputFormat;
if (outputFormat === "full") {
var outputObject = {
sEcho: this.echo,
iTotalRecords: fullTotal,
iTotalDisplayRecords: filteredTotal,
aaData: pagedData
};
return {...outputObject, ...customFacetResults};
}
else {
return pagedData;
}
}
}
common.DataTable = DataTable;
module.exports = common;