From 2ac96167916d39c98bdc5123f4a50be5fa460621 Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Fri, 22 Mar 2019 17:16:59 +0000 Subject: [PATCH 01/10] Tidying up the applyTransformation --- ditto/mappers/map.js | 32 ++++++++++++---------- test/ditto.js | 6 +++- test/mappings/test.js | 64 +++++++++++++++++++++++-------------------- test/results/test.js | 7 +++-- test/samples/test.js | 8 ++++++ 5 files changed, 70 insertions(+), 47 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 340fb1d..37a543f 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -33,9 +33,7 @@ async function map(document, mappings, plugins) { // here we check for that and construct the appropriate document and mappings objects to deal with it if (_.isArray(path)) { _.each(path, function(subPath){ - var subMapping = {}; - subMapping[key] = subPath; - return processMappings(subMapping, document, result); + return processMappings({[key]: subPath}, document, result); }); } @@ -143,7 +141,7 @@ async function map(document, mappings, plugins) { * the function name is anything after the @ and the () if exists * the paramteres are anything inside the () separated by a | */ - let paramteresArray, paramteresValues = []; + let paramteresValues = []; // Regular expression to extract any text between () let functionParameteres = path.match(/.+?\((.*)\)/); @@ -154,13 +152,14 @@ async function map(document, mappings, plugins) { if (!!functionParameteres) { // We need to check if the function parameters have inlined functions that we need to execute - paramteresArray = _.compact(functionParameteres[1].split('|')); + const paramteresArray = _.compact(functionParameteres[1].split('|')); + const _defaultValue = applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value); - if (_.last(paramteresArray).includes('*') && !! applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value)) { - return applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value) + if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { + return _defaultValue; } else { // we compact the array here to remove any undefined objects that are not caught by the _.get in the map function - paramteresValues = _.union(paramteresValues, _.map(paramteresArray, function(param){ return _.startsWith(param, '$') ? eval(param) : applyTransformation(key, param.replace(',', '|'), $key, $value) })); + paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(key, param.replace(',', '|'), $key, $value) }); } } @@ -171,7 +170,7 @@ async function map(document, mappings, plugins) { // Only execute the function if the parameters array is not empty if (!!_.compact(paramteresValues).length && plugins[functionCall]) { - return plugins[functionCall].apply(null, paramteresValues); + return plugins[functionCall](...paramteresValues); } } else return getValue(path, $value); @@ -203,7 +202,9 @@ async function map(document, mappings, plugins) { if (path === '!') return document; - if (_.startsWith(path, '>>')) { + if (_.startsWith(path, '$')) { + return eval(path); + } else if (_.startsWith(path, '>>')) { return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); } else if (_.startsWith(path, '!')) { return _.get(result, path.replace('!', '')); @@ -216,9 +217,12 @@ async function map(document, mappings, plugins) { path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); // Run a comparison between the values, and if fails skip the current data - const firstValue = applyTransformation('', parameters.comparator, '', JSON.stringify(subDocument)); - const secondValue = applyTransformation('', parameters.condition, '', JSON.stringify(subDocument)) - let isValidValue = operation(parameters.comparison, firstValue, secondValue); + const firstValue = applyTransformation(null, parameters.comparator, null, JSON.stringify(subDocument)); + const secondValue = applyTransformation(null, parameters.condition, null, JSON.stringify(subDocument)); + + console.log("parameters", parameters); + const comparison = parameters.comparison.startsWith('!') ? '!==' : '==='; + const isValidValue = operation(parameters.comparison, firstValue, secondValue); return isValidValue ? applyTransformation(null, parameters.targetValue, null, subDocument) : null; } else { @@ -233,7 +237,7 @@ async function map(document, mappings, plugins) { * @param {*} value1 * @param {*} value2 */ - function operation(op, value1, value2){ + function operation(op, value1, value2) { switch(op){ case '===': return value1 === value2; diff --git a/test/ditto.js b/test/ditto.js index 3ba9cb9..ba44473 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -51,7 +51,11 @@ describe('ditto Interface', function(){ }); it('should be able to assign a new value based on an already mapped one', function(){ - assert.strictEqual(this.result.displayName, "Ahmad Assaf"); + assert.strictEqual(this.result.displayName, "Ahmad Ahmad Assaf"); + }); + + it('should be able to allow for duplicate values without flattening/compacting them', function(){ + assert.strictEqual(this.result.fullName, "Ahmad Ahmad Assaf"); }); it('should be able to apply a condition on an output path', function(){ diff --git a/test/mappings/test.js b/test/mappings/test.js index 601de22..92ee634 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -1,36 +1,37 @@ 'use strict'; module.exports = { - "name" : "firstName", - "default_name" : "nonExistingProperty||>>this_should_be_the_firstName", - "nickname" : "nickname||>>nickname_not_found", - "isNickNameFound" : "nickname||>>%false", - "isDynamicDefault" : "nickname||firstName", - "fullName" : "@concatName(firstName|lastName)", - "fullNameDefault" : "@concatName(firstName|*!fullName)", + "name": "firstName", + "default_name": "nonExistingProperty||>>this_should_be_the_firstName", + "nickname": "nickname||>>nickname_not_found", + "isNickNameFound": "nickname||>>%false", + "isDynamicDefault": "nickname||firstName", + "fullName": "@concatName(firstName|middleName|lastName)", "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", - "completeName" : "@concatName(firstName|!fullName)", - "displayName" : "!fullName", + "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", + "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", + "completeName": "@concatName(firstName|!fullName)", + "displayName": "!fullName", "email": { "value": "email" }, "links": "links", "social_links": [{ - "output" : [], + "output": [], "innerDocument": "!links", "required": ["value"], - "mappings" : { + "mappings": { "value": "$value", "type": ">>test", "order": "$key", "social": ">>%true" } - },{ - "output" : [], + }, { + "output": [], "innerDocument": "social", "required": ["value"], - "mappings" : { - "value" : "value", + "mappings": { + "value": "value", "service": "service", "type": ">>social" } @@ -70,26 +71,26 @@ module.exports = { "prerequisite": "!!innerResult.value", "mappings": { "service": "@getLinkService(value|service)", - "type" : "@getLinkType(value|@getLinkService(value,service))", - "value" : "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" + "type": "@getLinkType(value|@getLinkService(value,service))", + "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" } }, "social_links_objects": { - "output" : {}, + "output": {}, "innerDocument": "!links", "key": "@generateId($value)", - "mappings" : { + "mappings": { "value": "$value" } }, "experience_primary": { "values": { - "output" : {}, + "output": {}, "innerDocument": "!", - "key" : "@generateId(title|company)", - "mappings" : { - "id" : "@generateId(title|company)", - "role" : "title", + "key": "@generateId(title|company)", + "mappings": { + "id": "@generateId(title|company)", + "role": "title", "organisationName": "company" } } @@ -98,10 +99,10 @@ module.exports = { "output": [], "innerDocument": "work", "mappings": { - "name" : "companyName", - "role" : "title", + "name": "companyName", + "role": "title", "startDate": "@parseDate(startDate)", - "current" : "current" + "current": "current" } }, "primaryExperience": "!experience[0]", @@ -117,7 +118,7 @@ module.exports = { "innerDocument": "work", "key": "@generateId(companyName|title)", "mappings": { - "id": "@generateId(companyName|title)", + "id": "@generateId(companyName|title)", "name": "companyName", "role": "title", "startDate": "startDate", @@ -125,6 +126,11 @@ module.exports = { } } }, + "volunteer": { + "output": [], + "innerDocument": "volunteer", + "value": "@concatName(organisation|>> at |title)" + }, "education": { "output": [], "innerDocument": "json.education", @@ -144,5 +150,5 @@ module.exports = { "universityName": "universityName" } }, - "primaryPhoto": "@createURL(>>http://photo.com/|!fullName)" + "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" } diff --git a/test/results/test.js b/test/results/test.js index 07a6409..def1c1f 100644 --- a/test/results/test.js +++ b/test/results/test.js @@ -6,11 +6,12 @@ module.exports = { "nickname": "nickname_not_found", "isNickNameFound": false, "isDynamicDefault": "Ahmad", - "fullName": "Ahmad Assaf", + "fullName": "Ahmad Ahmad Assaf", "fullNameDefault": "Ahmad Assaf", "fullNameDefaultHardcoded": "default", - "completeName": "Ahmad Ahmad Assaf", - "displayName": "Ahmad Assaf", + "fullName_withNotFoundMiddle" : "Ahmad Assaf", + "completeName": "Ahmad Ahmad Ahmad Assaf", + "displayName": "Ahmad Ahmad Assaf", "email": { "value": "ahmad.a.assaf@gmail.com" }, diff --git a/test/samples/test.js b/test/samples/test.js index d6462fd..86f62fe 100644 --- a/test/samples/test.js +++ b/test/samples/test.js @@ -3,6 +3,7 @@ module.exports = { "firstName" : "Ahmad", "lastName": "Assaf", + "middleName": "Ahmad", "email" : "ahmad.a.assaf@gmail.com", "title": "Data Scientist", "company": "Beamery", @@ -85,6 +86,13 @@ module.exports = { "current": false } ], + "volunteer": [ + { + "organisation": "PandaBear", + "title": "Volunteer", + "years": 5 + } + ], "json": { "education" : { "telecomParisTech" : { From b17833fe842493bf4c7a9c6449bf3ea44202544f Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 00:45:07 +0000 Subject: [PATCH 02/10] First iteration of cleaning up the interface --- ditto/mappers/map.js | 230 ++++++++++-------------------------------- test/ditto.js | 5 +- test/index.js | 2 +- test/mappings/test.js | 125 +++++++---------------- 4 files changed, 98 insertions(+), 264 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 37a543f..5ce35e7 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const assert = require('assert'); /** * @function map @@ -9,195 +10,81 @@ const _ = require('lodash'); */ async function map(document, mappings, plugins) { - /** - * @function processMappings - * - * @description Process the mappings file and map it to the actual values - * Check if the key is static or dynamic assignment. Static assignment assigns directly the - * the hardcoded value into the result object after removing the extra >> signs - * Otherwise, check if the path of the object exists. If so, check if the values have to be - * added in an array (push) or just simple object assignment - * If a prerequisite is defined then we will only assign the value if the prerequisite has passed - * - * @param {Object} document the object document we want to map - * @param {Object} result the object representing the result file - * @param {Object} mappings the object presenting the mappings between target document and mapped one - */ - function processMappings(mappings, document, result, parentKey) { - - _.each(mappings, function(path, key) { - - key = parentKey ? `${parentKey}.${key}` : key; - - // The path can be multiple definitions for retreiving the same piece of information from multiple places - // here we check for that and construct the appropriate document and mappings objects to deal with it - if (_.isArray(path)) { - _.each(path, function(subPath){ - return processMappings({[key]: subPath}, document, result); - }); + function processMappings(document, mappings, output, isIterable = false) { + if (!mappings) return document; + else if (isIterable) { + return document.map(function(_document) { + return processMappings(_document, mappings); + }); + } else if (typeof mappings === 'string') { + return applyTransformation(document, mappings, output); + } else if (mappings.hasOwnProperty('output')) { + let innerDocument = _.get(document, mappings.innerDocument.replace(/.\*/, ''), null); + if (mappings.innerDocument.endsWith('*')) { + innerDocument = _.values(innerDocument); } - - // Check if the path presents a flat object with a value that can be accessible directly with an _.get - if (!_.isPlainObject(path)) { - let value = applyTransformation(key, path, document.$key, document.$value); - - // here we check if the parent key of the value is not defined and only define it at least one value is there - // this resolves the issue of having an empty object defined e.g. location: {} - // the check if (!_.isUndefined(value)) will make sure we have null values to be picked up - // by our postMapper rather than having if (!_.isUndefined(value) && !!value) :) - if (_.isString(value)) { - if (!!value.length) _.set(result, key, value); - } else if (!_.isUndefined(value) && !_.isNull(value)) _.set(result, key, value); - - } else { - - // Check if the object is a nested object of objects or array of objects or not - if (!path.output) { - // Instantiate the empty object in the desired key and pass it to the recursive function - return processMappings(path, document, result, key); - - } else { - - // Reaching here we now know for sure that we will be creating a set of objects (array or object of objects) - // Assign the result object with its correct type defined in the path output - if (!_.has(result, key)) _.set(result, key, path.output) - - _.each(applyTransformation('', path.innerDocument), function($value, $key) { - - // first we need to check if we will be pushing objects or just plain items to an array - // This can be done by checking if we define a mappings object or not - if (path.mappings) { - - var innerResult = {}; - var processingDocument = path.innerDocument === '!' ? document : _.merge(_.cloneDeep($value), {$value: $value, $key: $key}); - - processMappings(path.mappings, processingDocument, innerResult); - - if (_.isArray(path.required) && - _.find(path.required, requiredPath => _.isNil(_.get(innerResult, requiredPath)))){ - innerResult = null; - } - - parseMappings(result, innerResult, path, key, $value, $key); - - } else { - // reaching here means that we are pushing only to a flat array and not an object - if (_.startsWith(path.value, '@')) { - _.updateWith(result, key, function(theArray){ return applyTransformation(key, path.value) }, []); - return false; - } else return _.updateWith(result, key, function(theArray){ theArray.push($value[path.value]); return theArray }, []); - } - - // here we are breaking out of the each if we have defined the innerDocument as the parent document - if (path.innerDocument === '!') return false; + let _output = processMappings(innerDocument, mappings.mappings, output, true); + if (mappings.hasOwnProperty('required')) { + mappings.required.forEach(_required => { + _output = _output.filter(_result => { + return !!_result[_required]; }); - - function parseMappings(result, innerResult, path, key, $value, $key) { - // based on the type of the result [] or {} we will either push or assign with key - if (!!innerResult) { - if (_.isArray(_.get(result,key))) { - if (!!path.prerequisite) { - if (!!eval(path.prerequisite)) _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); - } else _.updateWith(result, key, function(theArray){ theArray.push(innerResult); return theArray }, []); - } else { - let fullPath = `${key}.${applyTransformation(key, path['key'], $key, $value)}`; - if (!!path.prerequisite) { - if (!!eval(path.prerequisite)) _.set(result, fullPath, innerResult); - } else _.set(result, fullPath, innerResult); - } - } - - // After assigning the innerResult we need to make sure that there are no requirements on it (e.g., unique array) - if (!!path.requirements) { - _.each(path.requirements, function(requirement){ - _.set(result, key, applyTransformation(key, requirement, $key, $value)); - }); - } - return; - } - } + }); } - }); - - /** @function applyTransformation - * @description Apply a tranformation function on a path - * - * @param {String} path the path to pass for the _.get to retrieve the value - * @param {String} key the key of the result object that will contain the new mapped value - */ - function applyTransformation(key, path, $key, $value) { + if (_.isPlainObject(mappings.output)) { + let keys = []; + _output.forEach(element => { keys.push(element.$key) && delete element['$$key'] }); + _output = _.zipObject(keys, _output); + } + return _output; + } else { + const output = {}; + _.each(mappings, (value, key) => { + if (mappings.hasOwnProperty(key)) { + let _output = processMappings(document, value, output); + output[key] = _output; + } + }) + return output; + } + function applyTransformation(document, path, output) { + console.log("applyTransformation =====>"); + console.log("document",document); + console.log("path",path); if (path.includes('??')) { - return getValue(path, $value); + return getValue(document, path, output); } else if (_.startsWith(path, '$')) { return eval(path); } else if (_.startsWith(path, '@!')) { return eval(path.replace('@!', '')); } else if (_.startsWith(path, '@')) { - /** - * The parts in the string function are split into: - * before the @ is the first paramteres passed to the function - * the function name is anything after the @ and the () if exists - * the paramteres are anything inside the () separated by a | - */ let paramteresValues = []; - // Regular expression to extract any text between () let functionParameteres = path.match(/.+?\((.*)\)/); let functionCall = path.split('(')[0].replace('@', ''); - // Now we want to split the paramteres by a | in case we pass more than one param - // We also need to check if we are assigning a default value for that function denoted by a * if (!!functionParameteres) { - // We need to check if the function parameters have inlined functions that we need to execute const paramteresArray = _.compact(functionParameteres[1].split('|')); - const _defaultValue = applyTransformation(key, _.last(paramteresArray).replace('*', '').replace(',', '|'), $key, $value); + const _defaultValue = applyTransformation(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output); if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { return _defaultValue; } else { - // we compact the array here to remove any undefined objects that are not caught by the _.get in the map function - paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(key, param.replace(',', '|'), $key, $value) }); + paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(document, param.replace(',', '|'), output) }); } } - // Extract the function call and the first parameter - // Now the paramteres array contains the PATH for each element we want to pass to the @function - // We need now to actually get the actual values of these paths - // If the getValues fails that means that we are passing a hardocded value, so pass it as is - - // Only execute the function if the parameters array is not empty if (!!_.compact(paramteresValues).length && plugins[functionCall]) { return plugins[functionCall](...paramteresValues); } - } else return getValue(path, $value); + } else return getValue(document, path, output); } - /** - * @function getValue - * @description This function will get the value using the _.get by inspecting the scope of the get - * The existence of the ! will define a local scope in the result object rather than the document - * - * The Flat structure can contain the following cases: - * - starts with !: This will denote that the contact on which the _.get will be on a previously extracted - * value in the result file - * - starts with >>: This means a hardcoded value e.g., >>test -> test - * - contains @: This means that a function will be applied on the value before the @ sign - * The functions first parameter will be the value before the @ and any additional parameters will be defined - * between the parenthesis e.g., name@strip(','') -- will call --> strip(name, ',') - * - contains %: This will denote a casting function to the value using eval - * e.g., >>%true -> will be a true as a boolean and not as a string - * - contains ?? means that a condition has to be applied before assigning the value. The condition to apply - * is the one defined after the -> - * - * @param {String} path the path to pass for the _.get to retrieve the value - * @return {Object} result the object representing the result file - */ - function getValue(path, subDocument) { - + function getValue(document, path, output) { if (!path) return; if (path === '!') return document; @@ -207,28 +94,26 @@ async function map(document, mappings, plugins) { } else if (_.startsWith(path, '>>')) { return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); } else if (_.startsWith(path, '!')) { - return _.get(result, path.replace('!', '')); + return _.get(output, path.replace('!', '')); } else if (/\|\|/.test(path) && !path.includes('??') ) { let pathWithDefault = path.split(/\|\|/); - return getValue(pathWithDefault[0], subDocument) || getValue(`${pathWithDefault[1]}`); + return getValue(document, pathWithDefault[0]) || getValue(`${pathWithDefault[1]}`); } else if (path.includes('??') ){ - // First we need to get the value the condition is checking against .. and get the main value only if it is truthy let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); - // Run a comparison between the values, and if fails skip the current data - const firstValue = applyTransformation(null, parameters.comparator, null, JSON.stringify(subDocument)); - const secondValue = applyTransformation(null, parameters.condition, null, JSON.stringify(subDocument)); + const firstValue = applyTransformation(document, parameters.comparator, output); + const secondValue = applyTransformation(document, parameters.condition, output); - console.log("parameters", parameters); - const comparison = parameters.comparison.startsWith('!') ? '!==' : '==='; const isValidValue = operation(parameters.comparison, firstValue, secondValue); - - return isValidValue ? applyTransformation(null, parameters.targetValue, null, subDocument) : null; + console.log("parameters", parameters); + console.log("firstValue", firstValue); + console.log("secondValue", secondValue); + console.log("isValidValue", isValidValue); + console.log("document", document); + return isValidValue ? applyTransformation(document, parameters.targetValue, output) : null; } else { - // Here we check if the subDocument is a string (which indicates we need to get the value from a sub-path) - // We check for it to be a string because otherwise it can be the index of an array in the _.map() - return _.isPlainObject(subDocument) ? _.get(subDocument, path) : _.get(document, path); + return _.get(document, path); } /** @@ -252,10 +137,7 @@ async function map(document, mappings, plugins) { } } } - - var output = {} - processMappings(mappings, document, output); - return output; + return processMappings(document, mappings); } module.exports = map; \ No newline at end of file diff --git a/test/ditto.js b/test/ditto.js index ba44473..9f75baf 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -26,10 +26,13 @@ describe('ditto Interface', function(){ before(function(){ return new ditto(dummyMappings, dummyPlugin).unify(dummySample).then((result) => { this.result = result; + console.log("*********************"); + console.log("\n\noutput\n\n", JSON.stringify(result, null, 2)); + console.log("*********************"); }); }); - it('should be able to map an object with a direct flat mapping', function(){ + it.only('should be able to map an object with a direct flat mapping', function(){ assert.strictEqual(this.result.name, dummyResult.name); }); diff --git a/test/index.js b/test/index.js index cada32f..a1983c5 100644 --- a/test/index.js +++ b/test/index.js @@ -1,2 +1,2 @@ require('./ditto'); -require('./socialServices'); \ No newline at end of file +// require('./socialServices'); \ No newline at end of file diff --git a/test/mappings/test.js b/test/mappings/test.js index 92ee634..d412501 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -16,85 +16,67 @@ module.exports = { "value": "email" }, "links": "links", - "social_links": [{ - "output": [], - "innerDocument": "!links", - "required": ["value"], + "website_addresses": { + "output": {}, + "innerDocument": "linksv2.values.*", + "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", "mappings": { - "value": "$value", - "type": ">>test", - "order": "$key", - "social": ">>%true" + "$$key": "id", + "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", + "type": ">>other", + "keys": "keys" } - }, { - "output": [], - "innerDocument": "social", - "required": ["value"], - "mappings": { - "value": "value", - "service": "service", - "type": ">>social" + }, + "social_links_objects": { + "output" : {}, + "innerDocument": "links", + "mappings" : { + "$$key": "@generateId(!)", + "value": "!" } - }], + }, + "experience_object": { + "values": { + "output": {}, + "innerDocument": "work", + "mappings": { + "$$key": "@generateId(companyName|title)", + "id": "@generateId(companyName|title)", + "name": "companyName", + "role": "title", + "startDate": "startDate", + "current": "current" + } + } + }, "website_addresses_keyless": { "output": [], - "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", + "innerDocument": "linksv2.values.*", + "required": ["value"], "mappings": { "value": "value??type#==#>>website", "type": ">>other", } }, - "website_addresses": { - "output": {}, - "innerDocument": "linksv2.values", - "key": "id", - "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", - "mappings": { - "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", - "type": ">>other", - "keys": "keys" - } - }, "social_media_addresses": { "output": [], - "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", - "requirements": ["@uniqueArray(!social_media_addresses|>>value)", "@transformTwitterHandle(!social_media_addresses)"], + "innerDocument": "linksv2.values.*", + "required": ["value"], + "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], "mappings": { "value": "value??type#==#>>social" } }, "messaging": { "output": [], - "innerDocument": "linksv2.values", - "prerequisite": "!!innerResult.value", + "innerDocument": "linksv2.values.*", + "required": ["value"], "mappings": { "service": "@getLinkService(value|service)", "type": "@getLinkType(value|@getLinkService(value,service))", "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" } }, - "social_links_objects": { - "output": {}, - "innerDocument": "!links", - "key": "@generateId($value)", - "mappings": { - "value": "$value" - } - }, - "experience_primary": { - "values": { - "output": {}, - "innerDocument": "!", - "key": "@generateId(title|company)", - "mappings": { - "id": "@generateId(title|company)", - "role": "title", - "organisationName": "company" - } - } - }, "experience": { "output": [], "innerDocument": "work", @@ -112,43 +94,10 @@ module.exports = { "innerDocument": "work", "value": "companyName" }, - "experience_object": { - "values": { - "output": {}, - "innerDocument": "work", - "key": "@generateId(companyName|title)", - "mappings": { - "id": "@generateId(companyName|title)", - "name": "companyName", - "role": "title", - "startDate": "startDate", - "current": "current" - } - } - }, "volunteer": { "output": [], "innerDocument": "volunteer", "value": "@concatName(organisation|>> at |title)" }, - "education": { - "output": [], - "innerDocument": "json.education", - "mappings": { - "universityName": "$key", - "degree": "degree", - "location": "location" - } - }, - "education_object": { - "output": {}, - "key": "@generateId($key|degree)", - "innerDocument": "json.education", - "mappings": { - "degree": "degree", - "location": "location", - "universityName": "universityName" - } - }, "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" } From 2b71fc28cdd7847a84013431d0d0bac059f9d1b7 Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 12:28:33 +0000 Subject: [PATCH 03/10] Fixing single object iteration and misc other fixes --- ditto/mappers/map.js | 45 ++++----- test/mappings/test.js | 221 ++++++++++++++++++++++++------------------ 2 files changed, 149 insertions(+), 117 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 5ce35e7..79acc38 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -10,20 +10,23 @@ const assert = require('assert'); */ async function map(document, mappings, plugins) { - function processMappings(document, mappings, output, isIterable = false) { + function processMappings(document, mappings, output, options = {isIterable: false}) { if (!mappings) return document; - else if (isIterable) { - return document.map(function(_document) { - return processMappings(_document, mappings); + else if (options.isIterable) { + let _output = []; + _.each(document, (_document, key) => { + _output.push(processMappings(_document, mappings, output, {isIterable: false, key})); }); + return _output; } else if (typeof mappings === 'string') { - return applyTransformation(document, mappings, output); + return applyTransformation(document, mappings, output, _.get(options, 'key', null)); } else if (mappings.hasOwnProperty('output')) { - let innerDocument = _.get(document, mappings.innerDocument.replace(/.\*/, ''), null); - if (mappings.innerDocument.endsWith('*')) { - innerDocument = _.values(innerDocument); + let _document, _output, options = {}; + if (mappings.hasOwnProperty('innerDocument')) { + _document = _.get(document, mappings.innerDocument.replace(/.\*/, ''), null); + options['isIterable'] = true; } - let _output = processMappings(innerDocument, mappings.mappings, output, true); + _output = processMappings(_document || document, mappings.mappings, output, options); if (mappings.hasOwnProperty('required')) { mappings.required.forEach(_required => { _output = _output.filter(_result => { @@ -33,7 +36,8 @@ async function map(document, mappings, plugins) { } if (_.isPlainObject(mappings.output)) { let keys = []; - _output.forEach(element => { keys.push(element.$key) && delete element['$$key'] }); + _output = Array.isArray(_output) ? _output : [_output]; + _output.forEach(element => { keys.push(element.$$key) && delete element['$$key'] }); _output = _.zipObject(keys, _output); } return _output; @@ -44,14 +48,14 @@ async function map(document, mappings, plugins) { let _output = processMappings(document, value, output); output[key] = _output; } - }) + }); return output; } - function applyTransformation(document, path, output) { - console.log("applyTransformation =====>"); - console.log("document",document); - console.log("path",path); + function applyTransformation(document, path, output, $key) { + // console.log("APPLY TRANSFORMATIOM =====>"); + // console.log("document", document); + // console.log("path", path); if (path.includes('??')) { return getValue(document, path, output); } else if (_.startsWith(path, '$')) { @@ -87,7 +91,9 @@ async function map(document, mappings, plugins) { function getValue(document, path, output) { if (!path) return; - if (path === '!') return document; + if (path === '!') { + return document; + } if (_.startsWith(path, '$')) { return eval(path); @@ -97,7 +103,7 @@ async function map(document, mappings, plugins) { return _.get(output, path.replace('!', '')); } else if (/\|\|/.test(path) && !path.includes('??') ) { let pathWithDefault = path.split(/\|\|/); - return getValue(document, pathWithDefault[0]) || getValue(`${pathWithDefault[1]}`); + return getValue(document, pathWithDefault[0]) || getValue(document, `${pathWithDefault[1]}`); } else if (path.includes('??') ){ let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); @@ -106,11 +112,6 @@ async function map(document, mappings, plugins) { const secondValue = applyTransformation(document, parameters.condition, output); const isValidValue = operation(parameters.comparison, firstValue, secondValue); - console.log("parameters", parameters); - console.log("firstValue", firstValue); - console.log("secondValue", secondValue); - console.log("isValidValue", isValidValue); - console.log("document", document); return isValidValue ? applyTransformation(document, parameters.targetValue, output) : null; } else { return _.get(document, path); diff --git a/test/mappings/test.js b/test/mappings/test.js index d412501..8abcce7 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -1,103 +1,134 @@ 'use strict'; module.exports = { - "name": "firstName", - "default_name": "nonExistingProperty||>>this_should_be_the_firstName", - "nickname": "nickname||>>nickname_not_found", - "isNickNameFound": "nickname||>>%false", - "isDynamicDefault": "nickname||firstName", - "fullName": "@concatName(firstName|middleName|lastName)", - "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", - "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", - "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", - "completeName": "@concatName(firstName|!fullName)", - "displayName": "!fullName", - "email": { - "value": "email" - }, - "links": "links", - "website_addresses": { - "output": {}, - "innerDocument": "linksv2.values.*", - "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", - "mappings": { - "$$key": "id", - "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", - "type": ">>other", - "keys": "keys" - } - }, - "social_links_objects": { - "output" : {}, - "innerDocument": "links", - "mappings" : { - "$$key": "@generateId(!)", - "value": "!" - } - }, - "experience_object": { - "values": { - "output": {}, - "innerDocument": "work", - "mappings": { - "$$key": "@generateId(companyName|title)", - "id": "@generateId(companyName|title)", - "name": "companyName", - "role": "title", - "startDate": "startDate", - "current": "current" - } - } - }, - "website_addresses_keyless": { - "output": [], - "innerDocument": "linksv2.values.*", - "required": ["value"], - "mappings": { - "value": "value??type#==#>>website", - "type": ">>other", - } - }, - "social_media_addresses": { - "output": [], - "innerDocument": "linksv2.values.*", - "required": ["value"], - "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], - "mappings": { - "value": "value??type#==#>>social" - } - }, - "messaging": { - "output": [], - "innerDocument": "linksv2.values.*", - "required": ["value"], - "mappings": { - "service": "@getLinkService(value|service)", - "type": "@getLinkType(value|@getLinkService(value,service))", - "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" - } - }, - "experience": { - "output": [], - "innerDocument": "work", - "mappings": { - "name": "companyName", - "role": "title", - "startDate": "@parseDate(startDate)", - "current": "current" - } - }, - "primaryExperience": "!experience[0]", - "primaryRole": "!experience[0].role", + // "name": "firstName", + // "default_name": "nonExistingProperty||>>this_should_be_the_firstName", + // "nickname": "nickname||>>nickname_not_found", + // "isNickNameFound": "nickname||>>%false", + // "isDynamicDefault": "nickname||firstName", + // "fullName": "@concatName(firstName|middleName|lastName)", + // "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", + // "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", + // "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", + // "completeName": "@concatName(firstName|!fullName)", + // "displayName": "!fullName", + // "email": { + // "value": "email" + // }, + // "links": "links", + // "messaging": { + // "output": [], + // "innerDocument": "linksv2.values.*", + // "required": ["value"], + // "mappings": { + // "service": "@getLinkService(value|service)", + // "type": "@getLinkType(value|@getLinkService(value,service))", + // "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" + // } + // }, + // "website_addresses_keyless": { + // "output": [], + // "innerDocument": "linksv2.values.*", + // "required": ["value"], + // "mappings": { + // "value": "value??type#==#>>website", + // "type": ">>other", + // } + // }, + // "website_addresses": { + // "output": {}, + // "innerDocument": "linksv2.values.*", + // "required": ["value"], + // "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", + // "mappings": { + // "$$key": "id", + // "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", + // "type": ">>other", + // "keys": "keys" + // } + // }, + // "social_media_addresses": { + // "output": [], + // "innerDocument": "linksv2.values.*", + // "required": ["value"], + // "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], + // "mappings": { + // "value": "value??type#==#>>social" + // } + // }, + // "social_links_objects": { + // "output" : {}, + // "innerDocument": "links", + // "mappings" : { + // "$$key": "@generateId(!)", + // "value": "!" + // } + // }, + // "experience": { + // "output": [], + // "innerDocument": "work", + // "mappings": { + // "name": "companyName", + // "role": "title", + // "startDate": "@parseDate(startDate)", + // "current": "current" + // } + // }, + // "experience_primary": { + // "values": { + // "output" : {}, + // "mappings" : { + // "$$key" : "@generateId(title|company)", + // "id" : "@generateId(title|company)", + // "role" : "title", + // "organisationName": "company" + // } + // } + // }, + // "primaryExperience": "!experience[0]", + // "primaryRole": "!experience[0].role", "experiences": { "output": [], "innerDocument": "work", - "value": "companyName" - }, - "volunteer": { - "output": [], - "innerDocument": "volunteer", - "value": "@concatName(organisation|>> at |title)" + "$value": "companyName" }, - "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" + // "experience_object": { + // "values": { + // "output": {}, + // "innerDocument": "work", + // "mappings": { + // "$$key": "@generateId(companyName|title)", + // "id": "@generateId(companyName|title)", + // "name": "companyName", + // "role": "title", + // "startDate": "startDate", + // "current": "current" + // } + // } + // }, + // "education": { + // "output": [], + // "innerDocument": "json.education", + // "mappings": { + // "universityName": "$key", + // "degree": "degree", + // "location": "location" + // } + // }, + // "education_object": { + // "output": {}, + // "innerDocument": "json.education.*", + // "mappings": { + // "$$key": "@generateId($key|degree)", + // "degree": "degree", + // "location": "location", + // "universityName": "universityName" + // } + // }, + // // "volunteer": { + // // "output": [], + // // "innerDocument": "volunteer", + // // "value": "@concatName(organisation|>> at |title)" + // // }, + // "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" } From 0b9771bfffcd99e7e66b0c8e2a73c408c1a37737 Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 13:39:08 +0000 Subject: [PATCH 04/10] Various fixes on the before attempting to fix multiple mappings destinations --- ditto/mappers/map.js | 8 +- test/ditto.js | 2 +- test/mappings/test.js | 271 +++++++++++++++++++++++------------------- 3 files changed, 152 insertions(+), 129 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 79acc38..f21769d 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -40,12 +40,15 @@ async function map(document, mappings, plugins) { _output.forEach(element => { keys.push(element.$$key) && delete element['$$key'] }); _output = _.zipObject(keys, _output); } + if (mappings.hasOwnProperty('$push')) { + _output = _output.map($ => {return $.$value}) + } return _output; } else { const output = {}; _.each(mappings, (value, key) => { if (mappings.hasOwnProperty(key)) { - let _output = processMappings(document, value, output); + let _output = processMappings(document, value, output, options); output[key] = _output; } }); @@ -53,9 +56,6 @@ async function map(document, mappings, plugins) { } function applyTransformation(document, path, output, $key) { - // console.log("APPLY TRANSFORMATIOM =====>"); - // console.log("document", document); - // console.log("path", path); if (path.includes('??')) { return getValue(document, path, output); } else if (_.startsWith(path, '$')) { diff --git a/test/ditto.js b/test/ditto.js index 9f75baf..2e73956 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -32,7 +32,7 @@ describe('ditto Interface', function(){ }); }); - it.only('should be able to map an object with a direct flat mapping', function(){ + it('should be able to map an object with a direct flat mapping', function(){ assert.strictEqual(this.result.name, dummyResult.name); }); diff --git a/test/mappings/test.js b/test/mappings/test.js index 8abcce7..42657cb 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -1,134 +1,157 @@ 'use strict'; module.exports = { - // "name": "firstName", - // "default_name": "nonExistingProperty||>>this_should_be_the_firstName", - // "nickname": "nickname||>>nickname_not_found", - // "isNickNameFound": "nickname||>>%false", - // "isDynamicDefault": "nickname||firstName", - // "fullName": "@concatName(firstName|middleName|lastName)", - // "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", - // "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", - // "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", - // "completeName": "@concatName(firstName|!fullName)", - // "displayName": "!fullName", - // "email": { - // "value": "email" - // }, - // "links": "links", - // "messaging": { - // "output": [], - // "innerDocument": "linksv2.values.*", - // "required": ["value"], - // "mappings": { - // "service": "@getLinkService(value|service)", - // "type": "@getLinkType(value|@getLinkService(value,service))", - // "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" - // } - // }, - // "website_addresses_keyless": { - // "output": [], - // "innerDocument": "linksv2.values.*", - // "required": ["value"], - // "mappings": { - // "value": "value??type#==#>>website", - // "type": ">>other", - // } - // }, - // "website_addresses": { - // "output": {}, - // "innerDocument": "linksv2.values.*", - // "required": ["value"], - // "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", - // "mappings": { - // "$$key": "id", - // "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", - // "type": ">>other", - // "keys": "keys" - // } - // }, - // "social_media_addresses": { - // "output": [], - // "innerDocument": "linksv2.values.*", - // "required": ["value"], - // "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], - // "mappings": { - // "value": "value??type#==#>>social" - // } - // }, - // "social_links_objects": { - // "output" : {}, - // "innerDocument": "links", - // "mappings" : { - // "$$key": "@generateId(!)", - // "value": "!" - // } - // }, - // "experience": { - // "output": [], - // "innerDocument": "work", - // "mappings": { - // "name": "companyName", - // "role": "title", - // "startDate": "@parseDate(startDate)", - // "current": "current" - // } - // }, - // "experience_primary": { - // "values": { - // "output" : {}, - // "mappings" : { - // "$$key" : "@generateId(title|company)", - // "id" : "@generateId(title|company)", - // "role" : "title", - // "organisationName": "company" - // } - // } - // }, - // "primaryExperience": "!experience[0]", - // "primaryRole": "!experience[0].role", + "name": "firstName", + "default_name": "nonExistingProperty||>>this_should_be_the_firstName", + "nickname": "nickname||>>nickname_not_found", + "isNickNameFound": "nickname||>>%false", + "isDynamicDefault": "nickname||firstName", + "fullName": "@concatName(firstName|middleName|lastName)", + "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", + "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", + "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", + "completeName": "@concatName(firstName|!fullName)", + "displayName": "!fullName", + "email": { + "value": "email" + }, + "links": "links", + "social_links": [{ + "output" : [], + "innerDocument": "links", + "required": ["value"], + "mappings" : { + "value": "!", + "type": ">>test", + "order": "$key", + "social": ">>%true" + } + },{ + "output" : [], + "innerDocument": "social", + "required": ["value"], + "mappings" : { + "value" : "value", + "service": "service", + "type": ">>social" + } + }], + "messaging": { + "output": [], + "innerDocument": "linksv2.values.*", + "required": ["value"], + "mappings": { + "service": "@getLinkService(value|service)", + "type": "@getLinkType(value|@getLinkService(value,service))", + "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" + } + }, + "website_addresses_keyless": { + "output": [], + "innerDocument": "linksv2.values.*", + "required": ["value"], + "mappings": { + "value": "value??type#==#>>website", + "type": ">>other", + } + }, + "website_addresses": { + "output": {}, + "innerDocument": "linksv2.values.*", + "required": ["value"], + "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", + "mappings": { + "$$key": "id", + "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", + "type": ">>other", + "keys": "keys" + } + }, + "social_media_addresses": { + "output": [], + "innerDocument": "linksv2.values.*", + "required": ["value"], + "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], + "mappings": { + "value": "value??type#==#>>social" + } + }, + "social_links_objects": { + "output" : {}, + "innerDocument": "links", + "mappings" : { + "$$key": "@generateId(!)", + "value": "!" + } + }, + "experience": { + "output": [], + "innerDocument": "work", + "mappings": { + "name": "companyName", + "role": "title", + "startDate": "@parseDate(startDate)", + "current": "current" + } + }, + "experience_primary": { + "values": { + "output" : {}, + "mappings" : { + "$$key" : "@generateId(title|company)", + "id" : "@generateId(title|company)", + "role" : "title", + "organisationName": "company" + } + } + }, + "primaryExperience": "!experience[0]", + "primaryRole": "!experience[0].role", "experiences": { "output": [], "innerDocument": "work", - "$value": "companyName" + "$push": true, + "mappings": { + "$value": "companyName" + } }, - // "experience_object": { - // "values": { - // "output": {}, - // "innerDocument": "work", - // "mappings": { - // "$$key": "@generateId(companyName|title)", - // "id": "@generateId(companyName|title)", - // "name": "companyName", - // "role": "title", - // "startDate": "startDate", - // "current": "current" - // } - // } - // }, - // "education": { + "experience_object": { + "values": { + "output": {}, + "innerDocument": "work", + "mappings": { + "$$key": "@generateId(companyName|title)", + "id": "@generateId(companyName|title)", + "name": "companyName", + "role": "title", + "startDate": "startDate", + "current": "current" + } + } + }, + "education": { + "output": [], + "innerDocument": "json.education.*", + "mappings": { + "universityName": "$key", + "degree": "degree", + "location": "location" + } + }, + "education_object": { + "output": {}, + "innerDocument": "json.education.*", + "mappings": { + "$$key": "@generateId($key|degree)", + "degree": "degree", + "location": "location", + "universityName": "universityName" + } + }, + // "volunteer": { // "output": [], - // "innerDocument": "json.education", - // "mappings": { - // "universityName": "$key", - // "degree": "degree", - // "location": "location" - // } - // }, - // "education_object": { - // "output": {}, - // "innerDocument": "json.education.*", - // "mappings": { - // "$$key": "@generateId($key|degree)", - // "degree": "degree", - // "location": "location", - // "universityName": "universityName" - // } + // "innerDocument": "volunteer", + // "value": "@concatName(organisation|>> at |title)" // }, - // // "volunteer": { - // // "output": [], - // // "innerDocument": "volunteer", - // // "value": "@concatName(organisation|>> at |title)" - // // }, - // "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" + "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" } From 064ea290935e243d4c73c159d285b99bbae94e37 Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 14:39:52 +0000 Subject: [PATCH 05/10] First iteration of mappings migration done --- ditto/mappers/map.js | 7 +++++- test/mappings/test.js | 52 ++++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index f21769d..3dd5af2 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -46,10 +46,15 @@ async function map(document, mappings, plugins) { return _output; } else { const output = {}; + const reducer = (input, fn) => { + return _.reduceRight(input,(accumulator, currentValue) => fn(accumulator,currentValue)) + }; _.each(mappings, (value, key) => { if (mappings.hasOwnProperty(key)) { let _output = processMappings(document, value, output, options); - output[key] = _output; + if (Array.isArray(value)) { + output[key] = Array.isArray(value[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); + } else output[key] = _output; } }); return output; diff --git a/test/mappings/test.js b/test/mappings/test.js index 42657cb..0219a75 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -7,7 +7,7 @@ module.exports = { "isNickNameFound": "nickname||>>%false", "isDynamicDefault": "nickname||firstName", "fullName": "@concatName(firstName|middleName|lastName)", - "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", + "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", "completeName": "@concatName(firstName|!fullName)", @@ -16,22 +16,44 @@ module.exports = { "value": "email" }, "links": "links", + "social_links_objectified": [{ + "output": {}, + "innerDocument": "links", + "required": ["value"], + "mappings": { + "$$key": "!", + "value": "!", + "type": ">>test", + "order": "$key", + "social": ">>%true" + } + }, { + "output": {}, + "innerDocument": "social", + "required": ["value"], + "mappings": { + "$$key": "value", + "value": "value", + "service": "service", + "type": ">>social" + } + }], "social_links": [{ - "output" : [], + "output": [], "innerDocument": "links", "required": ["value"], - "mappings" : { + "mappings": { "value": "!", "type": ">>test", "order": "$key", "social": ">>%true" } - },{ - "output" : [], + }, { + "output": [], "innerDocument": "social", "required": ["value"], - "mappings" : { - "value" : "value", + "mappings": { + "value": "value", "service": "service", "type": ">>social" } @@ -77,9 +99,9 @@ module.exports = { } }, "social_links_objects": { - "output" : {}, + "output": {}, "innerDocument": "links", - "mappings" : { + "mappings": { "$$key": "@generateId(!)", "value": "!" } @@ -96,11 +118,11 @@ module.exports = { }, "experience_primary": { "values": { - "output" : {}, - "mappings" : { - "$$key" : "@generateId(title|company)", - "id" : "@generateId(title|company)", - "role" : "title", + "output": {}, + "mappings": { + "$$key": "@generateId(title|company)", + "id": "@generateId(title|company)", + "role": "title", "organisationName": "company" } } @@ -121,7 +143,7 @@ module.exports = { "innerDocument": "work", "mappings": { "$$key": "@generateId(companyName|title)", - "id": "@generateId(companyName|title)", + "id": "@generateId(companyName|title)", "name": "companyName", "role": "title", "startDate": "startDate", From 052ac7d46dd46077c66c48b09ac796b5235c9abd Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 15:40:21 +0000 Subject: [PATCH 06/10] Complete unit tests passing for the interface --- ditto/mappers/map.js | 19 +++++++++++-------- test/ditto.js | 10 ++-------- test/mappings/test.js | 4 ++-- test/results/test.js | 30 ++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 3dd5af2..e8b397e 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -1,5 +1,4 @@ const _ = require('lodash'); -const assert = require('assert'); /** * @function map @@ -47,14 +46,20 @@ async function map(document, mappings, plugins) { } else { const output = {}; const reducer = (input, fn) => { - return _.reduceRight(input,(accumulator, currentValue) => fn(accumulator,currentValue)) + return _.reduce(input,(accumulator, currentValue) => fn(accumulator,currentValue)) }; _.each(mappings, (value, key) => { if (mappings.hasOwnProperty(key)) { let _output = processMappings(document, value, output, options); if (Array.isArray(value)) { - output[key] = Array.isArray(value[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); - } else output[key] = _output; + _output = Array.isArray(value[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); + } + if (value.hasOwnProperty('requirements')) { + _.each(value.requirements, requirement => { + _output = applyTransformation(_output, requirement); + }); + } + if (!_.isNil(_output)) output[key] = _output; } }); return output; @@ -73,16 +78,15 @@ async function map(document, mappings, plugins) { let functionParameteres = path.match(/.+?\((.*)\)/); let functionCall = path.split('(')[0].replace('@', ''); - if (!!functionParameteres) { const paramteresArray = _.compact(functionParameteres[1].split('|')); - const _defaultValue = applyTransformation(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output); + const _defaultValue = applyTransformation(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output, $key); if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { return _defaultValue; } else { - paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(document, param.replace(',', '|'), output) }); + paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(document, param.replace(',', '|'), output, $key) }); } } @@ -115,7 +119,6 @@ async function map(document, mappings, plugins) { const firstValue = applyTransformation(document, parameters.comparator, output); const secondValue = applyTransformation(document, parameters.condition, output); - const isValidValue = operation(parameters.comparison, firstValue, secondValue); return isValidValue ? applyTransformation(document, parameters.targetValue, output) : null; } else { diff --git a/test/ditto.js b/test/ditto.js index 2e73956..639fd2b 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -15,20 +15,14 @@ describe('ditto Interface', function(){ let dummyMappings = require('./mappings/test'); let dummyPlugin = { - transformTwitterHandle: function transformTwitterHandle(target){ - _.each(target, function(link){ - link.value = "@ahmadaassaf" - }); - return target; + transformTwitterHandle: (target) => { + return `@${target.match(/^https?:\/\/(www\.)?twitter\.com\/(#!\/)?([^\/]+)(\/\w+)*$/)[3]}` } }; before(function(){ return new ditto(dummyMappings, dummyPlugin).unify(dummySample).then((result) => { this.result = result; - console.log("*********************"); - console.log("\n\noutput\n\n", JSON.stringify(result, null, 2)); - console.log("*********************"); }); }); diff --git a/test/mappings/test.js b/test/mappings/test.js index 0219a75..116ac6b 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -93,9 +93,9 @@ module.exports = { "output": [], "innerDocument": "linksv2.values.*", "required": ["value"], - "requirements": ["@uniqueArray(social_media_addresses|>>value)", "@transformTwitterHandle(value)"], + "requirements": ["@uniqueArray(!|>>value)"], "mappings": { - "value": "value??type#==#>>social" + "value": "@transformTwitterHandle(value)??service#==#>>twitter" } }, "social_links_objects": { diff --git a/test/results/test.js b/test/results/test.js index def1c1f..1bbca1a 100644 --- a/test/results/test.js +++ b/test/results/test.js @@ -49,6 +49,36 @@ module.exports = { "service": "twitter", "type": "social" }], + "social_links_objectified": { + "http://a.com": { + "value": "http://a.com", + "type": "test", + "order": 0, + "social": true + }, + "http://b.com": { + "value": "http://b.com", + "type": "test", + "order": 1, + "social": true + }, + "http://c.com": { + "value": "http://c.com", + "type": "test", + "order": 2, + "social": true + }, + "http://github.com/ahmadassaf": { + "value": "http://github.com/ahmadassaf", + "service": "github", + "type": "social" + }, + "http://twitter.com/ahmadaassaf": { + "value": "http://twitter.com/ahmadaassaf", + "service": "twitter", + "type": "social" + } + }, "website_addresses_keyless": [{ "value": "https://gravatar.com/ahmadassaf", "type": "other" From bf6894bda104e98ab6ccc0ea176f2f75d082a22f Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Mon, 25 Mar 2019 23:15:24 +0000 Subject: [PATCH 07/10] Extract the transform function --- ditto/mappers/lib/extract.js | 4 + ditto/mappers/lib/transform.js | 96 +++++++++++++++++++++++ ditto/mappers/map.js | 138 +++++++-------------------------- ditto/mappers/postmap.js | 36 ++++++++- test/mappings/test.js | 26 +++---- 5 files changed, 178 insertions(+), 122 deletions(-) create mode 100644 ditto/mappers/lib/extract.js create mode 100644 ditto/mappers/lib/transform.js diff --git a/ditto/mappers/lib/extract.js b/ditto/mappers/lib/extract.js new file mode 100644 index 0000000..79548a1 --- /dev/null +++ b/ditto/mappers/lib/extract.js @@ -0,0 +1,4 @@ +const _ = require('lodash'); +const transform = require('./transform'); + +module.exports = extract; \ No newline at end of file diff --git a/ditto/mappers/lib/transform.js b/ditto/mappers/lib/transform.js new file mode 100644 index 0000000..18c0cc1 --- /dev/null +++ b/ditto/mappers/lib/transform.js @@ -0,0 +1,96 @@ + +const _ = require('lodash'); + +module.exports = class Transformer { + + constructor(plugins){ + this.plugins = plugins; + } + + transform (document, path, output, $key) { + + const transformer = this; + + if (path.includes('??')) { + return transformer.extract(document, path, output); + } else if (_.startsWith(path, '$')) { + return eval(path); + } else if (_.startsWith(path, '@!')) { + return eval(path.replace('@!', '')); + } else if (_.startsWith(path, '@')) { + + let paramteresValues = []; + + let functionParameteres = path.match(/.+?\((.*)\)/); + let functionCall = path.split('(')[0].replace('@', ''); + if (!!functionParameteres) { + + const paramteresArray = _.compact(functionParameteres[1].split('|')); + const _defaultValue = transformer.transform(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output, $key); + + if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { + return _defaultValue; + } else { + paramteresValues = _.map(paramteresArray, function(param){ return transformer.transform(document, param.replace(',', '|'), output, $key) }); + } + } + + if (!!_.compact(paramteresValues).length && transformer.plugins[functionCall]) { + return transformer.plugins[functionCall](...paramteresValues); + } + + } else return transformer.extract(document, path, output); + } + + extract (document, path, output) { + + const transformer = this; + + if (!path) return; + + if (path === '!') return document; + + if (_.startsWith(path, '$')) { + return eval(path); + } else if (_.startsWith(path, '>>')) { + return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); + } else if (_.startsWith(path, '!')) { + return _.get(output, path.replace('!', '')); + } else if (/\|\|/.test(path) && !path.includes('??') ) { + let pathWithDefault = path.split(/\|\|/); + return transformer.extract(document, pathWithDefault[0]) || transformer.extract(document, `${pathWithDefault[1]}`); + } else if (path.includes('??') ){ + let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], + path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); + + const firstValue = transformer.transform(document, parameters.comparator, output); + const secondValue = transformer.transform(document, parameters.condition, output); + + const isValidValue = operation(parameters.comparison, firstValue, secondValue); + + return isValidValue ? transformer.transform(document, parameters.targetValue, output) : null; + + } else return _.get(document, path); + + /** + * Runs a comparison ( === , ==, !==, != ) against the given values + * @param {string} op + * @param {*} value1 + * @param {*} value2 + */ + function operation(op, value1, value2) { + switch(op){ + case '===': + return value1 === value2; + case '==': + return value1 == value2; + case '!==': + return value1 !== value2; + case '!=': + return value1 != value2; + } + return false; + } + } + +} \ No newline at end of file diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index e8b397e..9f8f307 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const Transformer = require('./lib/transform'); /** * @function map @@ -9,143 +10,64 @@ const _ = require('lodash'); */ async function map(document, mappings, plugins) { + const transformer = new Transformer(plugins); function processMappings(document, mappings, output, options = {isIterable: false}) { + if (!mappings) return document; + else if (options.isIterable) { - let _output = []; - _.each(document, (_document, key) => { - _output.push(processMappings(_document, mappings, output, {isIterable: false, key})); + return _.map(document, (_document, key) => { + return processMappings(_document, mappings, output, {isIterable: false, key}); }); - return _output; } else if (typeof mappings === 'string') { - return applyTransformation(document, mappings, output, _.get(options, 'key', null)); + return transformer.transform(document, mappings, output, _.get(options, 'key')); } else if (mappings.hasOwnProperty('output')) { - let _document, _output, options = {}; + let _output, options = {}; if (mappings.hasOwnProperty('innerDocument')) { - _document = _.get(document, mappings.innerDocument.replace(/.\*/, ''), null); options['isIterable'] = true; } - _output = processMappings(_document || document, mappings.mappings, output, options); + _output = processMappings(_.get(document, mappings.innerDocument, document), mappings.mappings, output, options); if (mappings.hasOwnProperty('required')) { - mappings.required.forEach(_required => { - _output = _output.filter(_result => { - return !!_result[_required]; - }); - }); + _output = _.last(_.map(mappings.required, _required => { return _.filter(_output, _required) })); } if (_.isPlainObject(mappings.output)) { - let keys = []; - _output = Array.isArray(_output) ? _output : [_output]; - _output.forEach(element => { keys.push(element.$$key) && delete element['$$key'] }); - _output = _.zipObject(keys, _output); + const __output = _.flattenDeep([_output]); + _output = _.zipObject(_.map(__output, $ => { return $.$$key}), __output); } if (mappings.hasOwnProperty('$push')) { _output = _output.map($ => {return $.$value}) } + return _output; + } else { const output = {}; const reducer = (input, fn) => { return _.reduce(input,(accumulator, currentValue) => fn(accumulator,currentValue)) - }; - _.each(mappings, (value, key) => { - if (mappings.hasOwnProperty(key)) { - let _output = processMappings(document, value, output, options); - if (Array.isArray(value)) { - _output = Array.isArray(value[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); + } + _.each(mappings, (mapping, path) => { + if (mappings.hasOwnProperty(path)) { + if (mapping.hasOwnProperty('key')) { + mapping.mappings['$$key'] = mapping.key; } - if (value.hasOwnProperty('requirements')) { - _.each(value.requirements, requirement => { - _output = applyTransformation(_output, requirement); - }); + let _output = processMappings(document, mapping, output, options); + if (Array.isArray(mapping)) { + _output = Array.isArray(mapping[0].output) ? reducer(_output, _.concat) : reducer(_output, _.merge); } - if (!_.isNil(_output)) output[key] = _output; - } - }); - return output; - } - - function applyTransformation(document, path, output, $key) { - if (path.includes('??')) { - return getValue(document, path, output); - } else if (_.startsWith(path, '$')) { - return eval(path); - } else if (_.startsWith(path, '@!')) { - return eval(path.replace('@!', '')); - } else if (_.startsWith(path, '@')) { - - let paramteresValues = []; - - let functionParameteres = path.match(/.+?\((.*)\)/); - let functionCall = path.split('(')[0].replace('@', ''); - if (!!functionParameteres) { - - const paramteresArray = _.compact(functionParameteres[1].split('|')); - const _defaultValue = applyTransformation(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output, $key); - - if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { - return _defaultValue; - } else { - paramteresValues = _.map(paramteresArray, function(param){ return applyTransformation(document, param.replace(',', '|'), output, $key) }); + if (mapping.hasOwnProperty('requirements')) { + _.each(mapping.requirements, requirement => { + _output = transformer.transform(_output, requirement); + }); } - } - if (!!_.compact(paramteresValues).length && plugins[functionCall]) { - return plugins[functionCall](...paramteresValues); + if (!_.isNil(_output)) output[path] = _output; } + }); - } else return getValue(document, path, output); - } - - function getValue(document, path, output) { - if (!path) return; - - if (path === '!') { - return document; - } - - if (_.startsWith(path, '$')) { - return eval(path); - } else if (_.startsWith(path, '>>')) { - return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); - } else if (_.startsWith(path, '!')) { - return _.get(output, path.replace('!', '')); - } else if (/\|\|/.test(path) && !path.includes('??') ) { - let pathWithDefault = path.split(/\|\|/); - return getValue(document, pathWithDefault[0]) || getValue(document, `${pathWithDefault[1]}`); - } else if (path.includes('??') ){ - let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], - path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); - - const firstValue = applyTransformation(document, parameters.comparator, output); - const secondValue = applyTransformation(document, parameters.condition, output); - const isValidValue = operation(parameters.comparison, firstValue, secondValue); - return isValidValue ? applyTransformation(document, parameters.targetValue, output) : null; - } else { - return _.get(document, path); - } - - /** - * Runs a comparison ( === , ==, !==, != ) against the given values - * @param {string} op - * @param {*} value1 - * @param {*} value2 - */ - function operation(op, value1, value2) { - switch(op){ - case '===': - return value1 === value2; - case '==': - return value1 == value2; - case '!==': - return value1 !== value2; - case '!=': - return value1 != value2; - } - return false; - } + return output; } } + return processMappings(document, mappings); } diff --git a/ditto/mappers/postmap.js b/ditto/mappers/postmap.js index 218493a..29f8d5e 100644 --- a/ditto/mappers/postmap.js +++ b/ditto/mappers/postmap.js @@ -11,7 +11,41 @@ async function postMap(result) { function isEmptyObject(value, key) { return _.isPlainObject(value) && _.isEmpty(value) ? true : false; } - return _(result).omitBy(_.isUndefined).omitBy(_.isNull).omitBy(isEmptyObject).value(); + /** + * Remove all specified keys from an object, no matter how deep they are. + * The removal is done in place, so run it on a copy if you don't want to modify the original object. + * This function has no limit so circular objects will probably crash the browser + * + * @param obj The object from where you want to remove the keys + * @param keys An array of property names (strings) to remove + */ + function removeKeys(obj, keys) { + var index; + for (var prop in obj) { + // important check that this is objects own property not from prototype prop inherited + if (obj.hasOwnProperty(prop)) { + switch(typeof(obj[prop])) { + case 'string': + index = keys.indexOf(prop); + if(index > -1) { + delete obj[prop]; + } + break; + case 'object': + index = keys.indexOf(prop); + if (index > -1) { + delete obj[prop]; + } else { + removeKeys(obj[prop], keys); + } + break; + } + } + } + } + // Make sure we remove Ditto built in keys + removeKeys(result, ['$$key']); + return _(result).omitBy([_.isNil, isEmptyObject]).value(); } module.exports = postMap; \ No newline at end of file diff --git a/test/mappings/test.js b/test/mappings/test.js index 116ac6b..465b061 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -20,8 +20,8 @@ module.exports = { "output": {}, "innerDocument": "links", "required": ["value"], + "key": "!", "mappings": { - "$$key": "!", "value": "!", "type": ">>test", "order": "$key", @@ -31,8 +31,8 @@ module.exports = { "output": {}, "innerDocument": "social", "required": ["value"], + "key": "value", "mappings": { - "$$key": "value", "value": "value", "service": "service", "type": ">>social" @@ -60,7 +60,7 @@ module.exports = { }], "messaging": { "output": [], - "innerDocument": "linksv2.values.*", + "innerDocument": "linksv2.values", "required": ["value"], "mappings": { "service": "@getLinkService(value|service)", @@ -70,7 +70,7 @@ module.exports = { }, "website_addresses_keyless": { "output": [], - "innerDocument": "linksv2.values.*", + "innerDocument": "linksv2.values", "required": ["value"], "mappings": { "value": "value??type#==#>>website", @@ -79,11 +79,11 @@ module.exports = { }, "website_addresses": { "output": {}, - "innerDocument": "linksv2.values.*", + "innerDocument": "linksv2.values", "required": ["value"], "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", + "key": "id", "mappings": { - "$$key": "id", "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", "type": ">>other", "keys": "keys" @@ -91,7 +91,7 @@ module.exports = { }, "social_media_addresses": { "output": [], - "innerDocument": "linksv2.values.*", + "innerDocument": "linksv2.values", "required": ["value"], "requirements": ["@uniqueArray(!|>>value)"], "mappings": { @@ -101,8 +101,8 @@ module.exports = { "social_links_objects": { "output": {}, "innerDocument": "links", + "key": "@generateId(!)", "mappings": { - "$$key": "@generateId(!)", "value": "!" } }, @@ -119,8 +119,8 @@ module.exports = { "experience_primary": { "values": { "output": {}, + "key": "@generateId(title|company)", "mappings": { - "$$key": "@generateId(title|company)", "id": "@generateId(title|company)", "role": "title", "organisationName": "company" @@ -141,8 +141,8 @@ module.exports = { "values": { "output": {}, "innerDocument": "work", + "key": "@generateId(companyName|title)", "mappings": { - "$$key": "@generateId(companyName|title)", "id": "@generateId(companyName|title)", "name": "companyName", "role": "title", @@ -153,7 +153,7 @@ module.exports = { }, "education": { "output": [], - "innerDocument": "json.education.*", + "innerDocument": "json.education", "mappings": { "universityName": "$key", "degree": "degree", @@ -162,9 +162,9 @@ module.exports = { }, "education_object": { "output": {}, - "innerDocument": "json.education.*", + "innerDocument": "json.education", + "key": "@generateId($key|degree)", "mappings": { - "$$key": "@generateId($key|degree)", "degree": "degree", "location": "location", "universityName": "universityName" From 45bac6f2b8fce44bda6a07caf221b8f5752b7a3c Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Tue, 26 Mar 2019 15:52:38 +0000 Subject: [PATCH 08/10] Completed refactoring the extrac transform steps --- ditto/mappers/lib/comparator.js | 28 ++++++++++ ditto/mappers/lib/extract.js | 4 -- ditto/mappers/lib/extractor.js | 38 +++++++++++++ ditto/mappers/lib/hoover.js | 32 +++++++++++ ditto/mappers/lib/transform.js | 96 -------------------------------- ditto/mappers/lib/transformer.js | 36 ++++++++++++ ditto/mappers/map.js | 3 +- ditto/mappers/postmap.js | 28 +--------- test/ditto.js | 9 +-- test/mappings/test.js | 14 ++--- test/results/test.js | 6 +- 11 files changed, 154 insertions(+), 140 deletions(-) create mode 100644 ditto/mappers/lib/comparator.js delete mode 100644 ditto/mappers/lib/extract.js create mode 100644 ditto/mappers/lib/extractor.js create mode 100644 ditto/mappers/lib/hoover.js delete mode 100644 ditto/mappers/lib/transform.js create mode 100644 ditto/mappers/lib/transformer.js diff --git a/ditto/mappers/lib/comparator.js b/ditto/mappers/lib/comparator.js new file mode 100644 index 0000000..55d0f4a --- /dev/null +++ b/ditto/mappers/lib/comparator.js @@ -0,0 +1,28 @@ +module.exports = function comperator(operation, firstValue, secondValue) { + + // Make sure we always apply strict comparison + if (operation === '==' || operation === '!=') { + operation = operation.replace('=', '=='); + } + + switch(operation) { + case '===': + return firstValue === secondValue; + case '==': + return firstValue === secondValue; + case '!==': + return firstValue !== secondValue; + case '!=': + return firstValue !== secondValue; + case '>': + return firstValue > secondValue; + case '<': + return firstValue < secondValue; + case '<=': + return firstValue <= secondValue; + case '>=': + return firstValue >= secondValue; + } + return false; + +} \ No newline at end of file diff --git a/ditto/mappers/lib/extract.js b/ditto/mappers/lib/extract.js deleted file mode 100644 index 79548a1..0000000 --- a/ditto/mappers/lib/extract.js +++ /dev/null @@ -1,4 +0,0 @@ -const _ = require('lodash'); -const transform = require('./transform'); - -module.exports = extract; \ No newline at end of file diff --git a/ditto/mappers/lib/extractor.js b/ditto/mappers/lib/extractor.js new file mode 100644 index 0000000..9e9d5ed --- /dev/null +++ b/ditto/mappers/lib/extractor.js @@ -0,0 +1,38 @@ + +const _ = require('lodash'); +const comparator = require('./comparator'); + +module.exports = class Extractor { + + constructor(plugins, transformer){ + this.plugins = plugins; + this.transformer = transformer; + } + + extract (document, path, output) { + + const extractor = this; + + if (_.isNil(path)) return; + + if (path === '!') return document; + + if (_.startsWith(path, '>>')) { + return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); + } else if (_.startsWith(path, '!')) { + return _.get(output, path.replace('!', '')); + } else if (/\|\|/.test(path) && !path.includes('??') ) { + let pathWithDefault = path.split(/\|\|/); + return extractor.extract(document, pathWithDefault[0], output) || extractor.extract(document, `${pathWithDefault[1]}`, output); + } else if (path.includes('??') ){ + const parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], + path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); + const _comparator = extractor.transformer.transform(document, parameters.comparator, output); + const _condition = extractor.transformer.transform(document, parameters.condition, output); + + return comparator(parameters.comparison, _comparator, _condition) ? extractor.transformer.transform(document, parameters.targetValue, output) : null; + + } else return _.get(document, path); + } + +} \ No newline at end of file diff --git a/ditto/mappers/lib/hoover.js b/ditto/mappers/lib/hoover.js new file mode 100644 index 0000000..ba30ea7 --- /dev/null +++ b/ditto/mappers/lib/hoover.js @@ -0,0 +1,32 @@ +/** + * Remove all specified keys from an object, no matter how deep they are. + * The removal is done in place, so run it on a copy if you don't want to modify the original object. + * This function has no limit so circular objects will probably crash the browser + * + * @param obj The object from where you want to remove the keys + * @param keys An array of property names (strings) to remove + */ +module.exports = function hoover(obj, keys) { + var index; + for (var prop in obj) { + // check that this is objects own property not from prototype prop inherited + if (obj.hasOwnProperty(prop)) { + switch(typeof(obj[prop])) { + case 'string': + index = keys.indexOf(prop); + if(index > -1) { + delete obj[prop]; + } + break; + case 'object': + index = keys.indexOf(prop); + if (index > -1) { + delete obj[prop]; + } else { + hoover(obj[prop], keys); + } + break; + } + } + } +} \ No newline at end of file diff --git a/ditto/mappers/lib/transform.js b/ditto/mappers/lib/transform.js deleted file mode 100644 index 18c0cc1..0000000 --- a/ditto/mappers/lib/transform.js +++ /dev/null @@ -1,96 +0,0 @@ - -const _ = require('lodash'); - -module.exports = class Transformer { - - constructor(plugins){ - this.plugins = plugins; - } - - transform (document, path, output, $key) { - - const transformer = this; - - if (path.includes('??')) { - return transformer.extract(document, path, output); - } else if (_.startsWith(path, '$')) { - return eval(path); - } else if (_.startsWith(path, '@!')) { - return eval(path.replace('@!', '')); - } else if (_.startsWith(path, '@')) { - - let paramteresValues = []; - - let functionParameteres = path.match(/.+?\((.*)\)/); - let functionCall = path.split('(')[0].replace('@', ''); - if (!!functionParameteres) { - - const paramteresArray = _.compact(functionParameteres[1].split('|')); - const _defaultValue = transformer.transform(document, _.last(paramteresArray).replace('*', '').replace(',', '|'), output, $key); - - if ( _.last(paramteresArray).includes('*') && !!_defaultValue) { - return _defaultValue; - } else { - paramteresValues = _.map(paramteresArray, function(param){ return transformer.transform(document, param.replace(',', '|'), output, $key) }); - } - } - - if (!!_.compact(paramteresValues).length && transformer.plugins[functionCall]) { - return transformer.plugins[functionCall](...paramteresValues); - } - - } else return transformer.extract(document, path, output); - } - - extract (document, path, output) { - - const transformer = this; - - if (!path) return; - - if (path === '!') return document; - - if (_.startsWith(path, '$')) { - return eval(path); - } else if (_.startsWith(path, '>>')) { - return _.startsWith(path, '>>%') ? eval(path.replace('>>%', '')) : path.replace('>>', ''); - } else if (_.startsWith(path, '!')) { - return _.get(output, path.replace('!', '')); - } else if (/\|\|/.test(path) && !path.includes('??') ) { - let pathWithDefault = path.split(/\|\|/); - return transformer.extract(document, pathWithDefault[0]) || transformer.extract(document, `${pathWithDefault[1]}`); - } else if (path.includes('??') ){ - let parameters = _.zipObject(['source', 'targetValue', 'comparator', 'comparison', 'condition'], - path.match(/(.+?)\?\?(.+?)\#(.*)\#(.+)/)); - - const firstValue = transformer.transform(document, parameters.comparator, output); - const secondValue = transformer.transform(document, parameters.condition, output); - - const isValidValue = operation(parameters.comparison, firstValue, secondValue); - - return isValidValue ? transformer.transform(document, parameters.targetValue, output) : null; - - } else return _.get(document, path); - - /** - * Runs a comparison ( === , ==, !==, != ) against the given values - * @param {string} op - * @param {*} value1 - * @param {*} value2 - */ - function operation(op, value1, value2) { - switch(op){ - case '===': - return value1 === value2; - case '==': - return value1 == value2; - case '!==': - return value1 !== value2; - case '!=': - return value1 != value2; - } - return false; - } - } - -} \ No newline at end of file diff --git a/ditto/mappers/lib/transformer.js b/ditto/mappers/lib/transformer.js new file mode 100644 index 0000000..5b7d635 --- /dev/null +++ b/ditto/mappers/lib/transformer.js @@ -0,0 +1,36 @@ + +const _ = require('lodash'); + +const Extractor = require('./extractor'); + +module.exports = class Transformer { + + constructor(plugins){ + this.plugins = plugins; + this.extractor = new Extractor(plugins, this); + } + + transform (document, path, output, $key) { + + const transformer = this; + + if (path.includes('??')) { + return transformer.extractor.extract(document, path, output); + } else if (_.startsWith(path, '$')) { + return eval(path); + } else if (_.startsWith(path, '@')) { + + const parameters = _.zipObject(['name', 'arguments'], path.split(/\((.*)\)/).filter(Boolean)); + const functionCall = parameters.name.replace('@', ''); + const paramteresValues = _.map(parameters.arguments.split('|'), param => { + return transformer.transform(document, param.replace(',', '|'), output, $key) + }).filter(Boolean); + + if (paramteresValues.length && transformer.plugins[functionCall]) { + return transformer.plugins[functionCall](...paramteresValues); + } + + } + return transformer.extractor.extract(document, path, output); + } +} \ No newline at end of file diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 9f8f307..81d4c40 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -1,5 +1,6 @@ const _ = require('lodash'); -const Transformer = require('./lib/transform'); + +const Transformer = require('./lib/transformer'); /** * @function map diff --git a/ditto/mappers/postmap.js b/ditto/mappers/postmap.js index 29f8d5e..6ea18cd 100644 --- a/ditto/mappers/postmap.js +++ b/ditto/mappers/postmap.js @@ -1,4 +1,5 @@ const _ = require('lodash'); +const hoover = require('./lib/hoover'); /** * @function postMap @@ -19,32 +20,9 @@ async function postMap(result) { * @param obj The object from where you want to remove the keys * @param keys An array of property names (strings) to remove */ - function removeKeys(obj, keys) { - var index; - for (var prop in obj) { - // important check that this is objects own property not from prototype prop inherited - if (obj.hasOwnProperty(prop)) { - switch(typeof(obj[prop])) { - case 'string': - index = keys.indexOf(prop); - if(index > -1) { - delete obj[prop]; - } - break; - case 'object': - index = keys.indexOf(prop); - if (index > -1) { - delete obj[prop]; - } else { - removeKeys(obj[prop], keys); - } - break; - } - } - } - } + // Make sure we remove Ditto built in keys - removeKeys(result, ['$$key']); + hoover(result, ['$$key']); return _(result).omitBy([_.isNil, isEmptyObject]).value(); } diff --git a/test/ditto.js b/test/ditto.js index 639fd2b..df486b5 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -16,7 +16,8 @@ describe('ditto Interface', function(){ let dummyPlugin = { transformTwitterHandle: (target) => { - return `@${target.match(/^https?:\/\/(www\.)?twitter\.com\/(#!\/)?([^\/]+)(\/\w+)*$/)[3]}` + const twitterHandle = target.match(/^https?:\/\/(www\.)?twitter\.com\/(#!\/)?([^\/]+)(\/\w+)*$/); + return !!twitterHandle ? `@${twitterHandle[3]}` : null; } }; @@ -48,11 +49,11 @@ describe('ditto Interface', function(){ }); it('should be able to assign a new value based on an already mapped one', function(){ - assert.strictEqual(this.result.displayName, "Ahmad Ahmad Assaf"); + assert.strictEqual(this.result.displayName, "Ahmad Ahmad AbdelMuti Assaf"); }); it('should be able to allow for duplicate values without flattening/compacting them', function(){ - assert.strictEqual(this.result.fullName, "Ahmad Ahmad Assaf"); + assert.strictEqual(this.result.fullName, "Ahmad Ahmad AbdelMuti Assaf"); }); it('should be able to apply a condition on an output path', function(){ @@ -201,7 +202,7 @@ describe('ditto Interface', function(){ "prerequisite": "!!innerResult.value", "required": ["value"], "mappings": { - "value": "value??type#==#>>social" + "value": "value??type#===#>>social" } } }).unify(badObj).then((result) => { diff --git a/test/mappings/test.js b/test/mappings/test.js index 465b061..f98289a 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -6,10 +6,10 @@ module.exports = { "nickname": "nickname||>>nickname_not_found", "isNickNameFound": "nickname||>>%false", "isDynamicDefault": "nickname||firstName", - "fullName": "@concatName(firstName|middleName|lastName)", - "fullNameDefaultHardcoded": "@concatName(firstName|lastName|*>>default)", + "fullName": "@concatName(firstName|middleName||>>AbdelMuti|lastName)", + "fullNameDefaultHardcoded": "@concatName(nonExistingProperty)||>>default", "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", - "fullNameDefault": "@concatName(firstName|*!fullName_withNotFoundMiddle)", + "fullNameDefault": "!fullName_withNotFoundMiddle", "completeName": "@concatName(firstName|!fullName)", "displayName": "!fullName", "email": { @@ -65,7 +65,7 @@ module.exports = { "mappings": { "service": "@getLinkService(value|service)", "type": "@getLinkType(value|@getLinkService(value,service))", - "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#==#>>messaging" + "value": "@cleanURI(value|@getLinkType(value,@getLinkService(value,service)))??@getLinkType(value|@getLinkService(value,service))#===#>>messaging" } }, "website_addresses_keyless": { @@ -73,7 +73,7 @@ module.exports = { "innerDocument": "linksv2.values", "required": ["value"], "mappings": { - "value": "value??type#==#>>website", + "value": "value??type#===#>>website", "type": ">>other", } }, @@ -84,7 +84,7 @@ module.exports = { "prerequisite": "!!innerResult.value && !!innerResult.keys && !!innerResult.keys.length", "key": "id", "mappings": { - "value": "value??keys[0]#==#>>f5e32a6faaa7ead6ba201e8fa25733ee", + "value": "value??keys[0]#===#>>f5e32a6faaa7ead6ba201e8fa25733ee", "type": ">>other", "keys": "keys" } @@ -95,7 +95,7 @@ module.exports = { "required": ["value"], "requirements": ["@uniqueArray(!|>>value)"], "mappings": { - "value": "@transformTwitterHandle(value)??service#==#>>twitter" + "value": "@transformTwitterHandle(value)??service#===#>>twitter" } }, "social_links_objects": { diff --git a/test/results/test.js b/test/results/test.js index 1bbca1a..19839a1 100644 --- a/test/results/test.js +++ b/test/results/test.js @@ -6,12 +6,12 @@ module.exports = { "nickname": "nickname_not_found", "isNickNameFound": false, "isDynamicDefault": "Ahmad", - "fullName": "Ahmad Ahmad Assaf", + "fullName": "Ahmad Ahmad AbdelMuti Assaf", "fullNameDefault": "Ahmad Assaf", "fullNameDefaultHardcoded": "default", "fullName_withNotFoundMiddle" : "Ahmad Assaf", - "completeName": "Ahmad Ahmad Ahmad Assaf", - "displayName": "Ahmad Ahmad Assaf", + "completeName": "Ahmad Ahmad Ahmad AbdelMuti Assaf", + "displayName": "Ahmad Ahmad AbdelMuti Assaf", "email": { "value": "ahmad.a.assaf@gmail.com" }, From 593fa77e7b92820f809d71095bf901070ae9fbbe Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Tue, 26 Mar 2019 16:03:53 +0000 Subject: [PATCH 09/10] Adding support for native JavaScript functions --- ditto/mappers/lib/transformer.js | 2 ++ test/ditto.js | 1 + test/mappings/test.js | 6 ++++-- test/results/test.js | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ditto/mappers/lib/transformer.js b/ditto/mappers/lib/transformer.js index 5b7d635..828962f 100644 --- a/ditto/mappers/lib/transformer.js +++ b/ditto/mappers/lib/transformer.js @@ -18,6 +18,8 @@ module.exports = class Transformer { return transformer.extractor.extract(document, path, output); } else if (_.startsWith(path, '$')) { return eval(path); + } else if (_.startsWith(path, '@!')) { + return eval(path.replace('@!', '')); } else if (_.startsWith(path, '@')) { const parameters = _.zipObject(['name', 'arguments'], path.split(/\((.*)\)/).filter(Boolean)); diff --git a/test/ditto.js b/test/ditto.js index df486b5..96af189 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -24,6 +24,7 @@ describe('ditto Interface', function(){ before(function(){ return new ditto(dummyMappings, dummyPlugin).unify(dummySample).then((result) => { this.result = result; + console.log(JSON.stringify(result, null, 2)); }); }); diff --git a/test/mappings/test.js b/test/mappings/test.js index f98289a..a539564 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -1,4 +1,4 @@ -'use strict'; +const _ = require('lodash'); module.exports = { "name": "firstName", @@ -11,6 +11,7 @@ module.exports = { "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", "fullNameDefault": "!fullName_withNotFoundMiddle", "completeName": "@concatName(firstName|!fullName)", + "concatendatedName": _.snakeCase('!fullName'), "displayName": "!fullName", "email": { "value": "email" @@ -175,5 +176,6 @@ module.exports = { // "innerDocument": "volunteer", // "value": "@concatName(organisation|>> at |title)" // }, - "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)" + "primaryPhoto": "@createURL(>>http://photo.com/|!fullNameDefault)", + "createdAt": "@!new Date('2019').toISOString()" } diff --git a/test/results/test.js b/test/results/test.js index 19839a1..83f36aa 100644 --- a/test/results/test.js +++ b/test/results/test.js @@ -177,5 +177,6 @@ module.exports = { "universityName": "University of St.Andrews" } }, - "primaryPhoto": "http://photo.com/ahmadassaf" + "primaryPhoto": "http://photo.com/ahmadassaf", + "createdAt": "2019-01-01T00:00:00.000Z" } From 031e845b0c81b6f48414dfcc0f3b0e472ab8e9dc Mon Sep 17 00:00:00 2001 From: Ahmad Assaf Date: Tue, 26 Mar 2019 19:27:59 +0000 Subject: [PATCH 10/10] Finishing V2. overhaul with all tests passing and mappings updates --- ditto/mappers/map.js | 13 ++++--- ditto/mappers/postmap.js | 10 +----- test/ditto.js | 1 - test/index.js | 2 +- test/mappings/services/facebook.js | 37 ++++++++++++------- test/mappings/services/facebook_raw.js | 38 +++++++++++++------- test/mappings/services/github.js | 25 +++++++------ test/mappings/services/github_raw.js | 33 ++++++++++------- test/mappings/services/google.js | 23 +++++++----- test/mappings/services/google_raw.js | 24 ++++++++----- test/mappings/services/linkedin.js | 44 +++++++++++++++-------- test/mappings/services/linkedin_raw.js | 50 +++++++++++++++++--------- test/mappings/test.js | 1 - 13 files changed, 186 insertions(+), 115 deletions(-) diff --git a/ditto/mappers/map.js b/ditto/mappers/map.js index 81d4c40..d6bdf74 100644 --- a/ditto/mappers/map.js +++ b/ditto/mappers/map.js @@ -23,22 +23,21 @@ async function map(document, mappings, plugins) { } else if (typeof mappings === 'string') { return transformer.transform(document, mappings, output, _.get(options, 'key')); } else if (mappings.hasOwnProperty('output')) { - let _output, options = {}; + let _output, _innerDocument = '!', options = {}; if (mappings.hasOwnProperty('innerDocument')) { options['isIterable'] = true; + _innerDocument = mappings.innerDocument; } - _output = processMappings(_.get(document, mappings.innerDocument, document), mappings.mappings, output, options); + _output = processMappings(transformer.transform(document, _innerDocument, output), mappings.mappings, output, options); if (mappings.hasOwnProperty('required')) { - _output = _.last(_.map(mappings.required, _required => { return _.filter(_output, _required) })); + _output = _.last(_.map(mappings.required, _required => { return _.filter(_.flatten([_output]), _required) })); } if (_.isPlainObject(mappings.output)) { const __output = _.flattenDeep([_output]); _output = _.zipObject(_.map(__output, $ => { return $.$$key}), __output); + } else if (mappings.hasOwnProperty('$push')) { + _output = _.compact(_output.map($ => {return $.$value})); } - if (mappings.hasOwnProperty('$push')) { - _output = _output.map($ => {return $.$value}) - } - return _output; } else { diff --git a/ditto/mappers/postmap.js b/ditto/mappers/postmap.js index 6ea18cd..eaf913e 100644 --- a/ditto/mappers/postmap.js +++ b/ditto/mappers/postmap.js @@ -12,18 +12,10 @@ async function postMap(result) { function isEmptyObject(value, key) { return _.isPlainObject(value) && _.isEmpty(value) ? true : false; } - /** - * Remove all specified keys from an object, no matter how deep they are. - * The removal is done in place, so run it on a copy if you don't want to modify the original object. - * This function has no limit so circular objects will probably crash the browser - * - * @param obj The object from where you want to remove the keys - * @param keys An array of property names (strings) to remove - */ // Make sure we remove Ditto built in keys hoover(result, ['$$key']); - return _(result).omitBy([_.isNil, isEmptyObject]).value(); + return _(result).omitBy(_.isNil).omitBy(isEmptyObject).value(); } module.exports = postMap; \ No newline at end of file diff --git a/test/ditto.js b/test/ditto.js index 96af189..df486b5 100644 --- a/test/ditto.js +++ b/test/ditto.js @@ -24,7 +24,6 @@ describe('ditto Interface', function(){ before(function(){ return new ditto(dummyMappings, dummyPlugin).unify(dummySample).then((result) => { this.result = result; - console.log(JSON.stringify(result, null, 2)); }); }); diff --git a/test/index.js b/test/index.js index a1983c5..cada32f 100644 --- a/test/index.js +++ b/test/index.js @@ -1,2 +1,2 @@ require('./ditto'); -// require('./socialServices'); \ No newline at end of file +require('./socialServices'); \ No newline at end of file diff --git a/test/mappings/services/facebook.js b/test/mappings/services/facebook.js index f30f132..0ccc50e 100644 --- a/test/mappings/services/facebook.js +++ b/test/mappings/services/facebook.js @@ -19,7 +19,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile.profileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile.profileUrl)", @@ -32,8 +31,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -50,15 +52,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile.id)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile.id)", @@ -68,8 +72,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { @@ -88,8 +95,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -108,8 +118,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/facebook_raw.js b/test/mappings/services/facebook_raw.js index d5272cc..0ba0437 100644 --- a/test/mappings/services/facebook_raw.js +++ b/test/mappings/services/facebook_raw.js @@ -22,7 +22,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.link)", @@ -35,15 +34,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -53,8 +54,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { @@ -74,8 +78,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -94,14 +101,16 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output" : {}, - "innerDocument": "!", "key" : "@generateId(data.picture.data.url)", "required" : ["value"], "mappings" : { @@ -111,8 +120,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/github.js b/test/mappings/services/github.js index 6e1ab1d..868f70b 100644 --- a/test/mappings/services/github.js +++ b/test/mappings/services/github.js @@ -19,7 +19,6 @@ module.exports = { "values" : [{ "output": {}, "key": "@generateIdForLinks(data.profile.profileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile.profileUrl)", @@ -34,7 +33,6 @@ module.exports = { },{ "output": {}, "key": "@generateId(data.profile._json.blog)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.blog)", @@ -44,15 +42,17 @@ module.exports = { }], "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { "values" : { "output": {}, "key": "@generateId(data.profile._json.company)", - "innerDocument": "!", "required": ["organisationName"], "mappings" : { "id" : "@generateId(data.profile._json.company)", @@ -61,15 +61,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.email)", @@ -79,8 +81,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { diff --git a/test/mappings/services/github_raw.js b/test/mappings/services/github_raw.js index 10dbcb2..fcda236 100644 --- a/test/mappings/services/github_raw.js +++ b/test/mappings/services/github_raw.js @@ -19,7 +19,6 @@ module.exports = { "values" : [{ "output": {}, "key": "@generateIdForLinks(data.html_url)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.html_url)", @@ -33,7 +32,6 @@ module.exports = { } },{ "output": {}, - "innerDocument": "!", "key": "@generateId(data.blog)", "required": ["value"], "mappings" : { @@ -44,15 +42,17 @@ module.exports = { }], "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "experience": { "values" : { "output": {}, "key": "@generateId(data.company)", - "innerDocument": "!", "required": ["organisationName"], "mappings" : { "id" : "@generateId(data.company)", @@ -61,15 +61,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -79,15 +81,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.avatar_url)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.avatar_url)", @@ -96,8 +100,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { diff --git a/test/mappings/services/google.js b/test/mappings/services/google.js index 59023b8..22da5e2 100644 --- a/test/mappings/services/google.js +++ b/test/mappings/services/google.js @@ -21,7 +21,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile._json.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile._json.link)", @@ -34,15 +33,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.picture)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.picture)", @@ -52,8 +53,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -70,8 +74,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/google_raw.js b/test/mappings/services/google_raw.js index 5b248a1..11913cd 100644 --- a/test/mappings/services/google_raw.js +++ b/test/mappings/services/google_raw.js @@ -26,7 +26,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.link)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.link)", @@ -39,15 +38,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.picture)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.picture)", @@ -57,15 +58,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.email)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.email)", @@ -75,8 +78,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/linkedin.js b/test/mappings/services/linkedin.js index f601da9..7f09095 100644 --- a/test/mappings/services/linkedin.js +++ b/test/mappings/services/linkedin.js @@ -20,7 +20,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.profile._json.publicProfileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.profile._json.publicProfileUrl)", @@ -35,8 +34,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { @@ -53,8 +55,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { @@ -79,8 +84,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -102,8 +110,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "languages" : { @@ -119,15 +130,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!languages.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, "key": "@generateId(data.profile._json.pictureUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.profile._json.pictureUrl)", @@ -136,8 +149,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/services/linkedin_raw.js b/test/mappings/services/linkedin_raw.js index b37474a..13ebf1c 100644 --- a/test/mappings/services/linkedin_raw.js +++ b/test/mappings/services/linkedin_raw.js @@ -20,7 +20,6 @@ module.exports = { "values" : { "output": {}, "key": "@generateIdForLinks(data.publicProfileUrl)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateIdForLinks(data.publicProfileUrl)", @@ -35,15 +34,17 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!links.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "emails" : { "values" : { "output": {}, "key": "@generateId(data.emailAddress)", - "innerDocument": "!", "required": ["value"], "mappings" : { "id" : "@generateId(data.emailAddress)", @@ -53,8 +54,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!emails.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "location": { @@ -79,8 +83,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!experience.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "education" : { @@ -102,8 +109,11 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!education.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "languages" : { @@ -119,25 +129,31 @@ module.exports = { }, "keys" : { "output": [], - "innerDocument": "!languages.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "photos" : { "values" : { "output": {}, - "key": "@generateId($value)", + "key": "@generateId(!)", "innerDocument": "data.pictureUrls.values", "required": ["value"], "mappings" : { - "id" : "@generateId($value)", - "value": "$value" + "id" : "@generateId(!)", + "value": "!" } }, "keys" : { "output": [], - "innerDocument": "!photos.values", - "value": "id" + "innerDocument": "!values", + "$push": true, + "mappings": { + "$value": "id" + } } }, "createdAt": "@!new Date()", diff --git a/test/mappings/test.js b/test/mappings/test.js index a539564..9441af9 100644 --- a/test/mappings/test.js +++ b/test/mappings/test.js @@ -11,7 +11,6 @@ module.exports = { "fullName_withNotFoundMiddle": "@concatName(firstName|fullName.middleName|lastName)", "fullNameDefault": "!fullName_withNotFoundMiddle", "completeName": "@concatName(firstName|!fullName)", - "concatendatedName": _.snakeCase('!fullName'), "displayName": "!fullName", "email": { "value": "email"