We Keep Coding

JavaScript: Transform from one nested JSON format to another nested JSON format

I am trying to transform an AVRO schema into an ElasticSearch index template. Both are JSON structured with a few things to check while transforming. I tried using recursion to get all the nested elements out and then pair them with their parents but writing to a dictionary while parsing deep with recursion compelled me to ask this question.
So basically I have this AVRO schema file:
{
"name": "animal",
"type": [
"null",
{
"type": "record",
"name": "zooAnimals",
"fields": [{
"name": "color",
"type": ["null", "string"],
"default": null
},
{
"name": "skinType",
"type": ["null", "string"],
"default": null
},
{
"name": "species",
"type": {
"type": "record",
"name": "AnimalSpecies",
"fields": [{
"name": "terrestrial",
"type": "string"
},
{
"name": "aquatic",
"type": "string"
}
]
}
},
{
"name": "behavior",
"type": [
"null",
{
"type": "record",
"name": "AnimalBehaviors",
"fields": [{
"name": "sound",
"type": ["null", "string"],
"default": null
},
{
"name": "hunt",
"type": ["null", "string"],
"default": null
}
]
}
],
"default": null
}
]
}
]
}
and I would like it to get transformed into this (Elasticsearch index template format):
{
"properties": {
"color" :{
"type" : "keyword"
},
"skinType" :{
"type" : "keyword"
},
"species" :{
"properties" : {
"terrestrial" : {
"type" : "keyword"
},
"aquatic" : {
"type" : "keyword"
},
}
},
"behavior" : {
"properties" : {
"sound" : {
"type" : "keyword"
},
"hunt" : {
"type" : "keyword"
}
}
}
}
}
Important Notes: The nesting on the AVRO schema could be furthermore nested and that's why I was thinking recursion to solve. Also, the type of the type filed could be an Array or a Map as shown for behavior vs. species where behavior has an array and species has a map.
If you must see that I did my trial and error, here's my code that's not getting me anywhere:
const checkDataTypeFromObject = function (obj) {
if (Object.prototype.toString.call(obj) === "[object Array]") {
obj.map(function (item) {
if (Object.prototype.toString.call(item) === "[object Object]") {
// so this is an object that could contain further nested fields
dataType = item;
mappings.properties[item.name] = { "type" : item.type}
if (item.hasOwnProperty("fields")) {
checkDataTypeFromObject(item.fields);
} else if (item.hasOwnProperty("type")) {
checkDataTypeFromObject(item.type);
}
} else if (item === null) {
// discard the nulls, nothing to do here
} else {
// if not dict or null, this is the dataType we are looking for
dataType = item;
}
return item.name
});

We can break it down using inductive reasoning. The numbered points below correspond the the numbered comments in the code -
if the input, t, is null, return an empty object
(inductive) t is not null. If t.type is an object, transform each leaf and sum into a single object
(inductive) t is not null and t.type is not an object. If t.fields is an object, transform each leaf, assign to { [name]: ... }, and sum into a single properties object
(inductive) t is not null and t.type is not an object, and t.fields is not an object. Return keyword.
const transform = t =>
t === "null"
? {} // <- 1
: isObject(t.type)
? arr(t.type) // <- 2
.map(transform)
.reduce(assign, {})
: isObject(t.fields)
? { propertries: // <- 3
arr(t.fields)
.map(v => ({ [v.name]: transform(v) }))
.reduce(assign, {})
}
: { type: "keyword" } // <- 4
With a few helpers to keep complexity out of our way -
const assign = (t, u) =>
Object.assign(t, u)
const arr = t =>
Array.isArray(t) ? t : [t]
const isObject = t =>
Object(t) === t
Simply run the transform -
console.log(transform(input))
Expand the snippet below to verify the result in your browser -
const assign = (t, u) =>
Object.assign(t, u)
const arr = t =>
Array.isArray(t) ? t : [t]
const isObject = t =>
Object(t) === t
const transform = t =>
t === "null"
? {}
: isObject(t.type)
? arr(t.type)
.map(transform)
.reduce(assign, {})
: isObject(t.fields)
? { propertries:
arr(t.fields)
.map(v => ({ [v.name]: transform(v) }))
.reduce(assign, {})
}
: { type: "keyword" }
const input =
{name: "animal", type: ["null", {type: "record", name: "zooAnimals", fields: [{name: "color", type: ["null", "string"], default: null}, {name: "skinType", type: ["null", "string"], default: null}, {name: "species", type: {type: "record", name: "AnimalSpecies", fields: [{name: "terrestrial", type: "string"}, {name: "aquatic", type: "string"}]}}, {name: "behavior", type: ["null", {type: "record", name: "AnimalBehaviors", fields: [{name: "sound", type: ["null", "string"], default: null}, {name: "hunt", type: ["null", "string"], default: null}]}], default: null}]}]}
console.log(transform(input))
Output -
{
"propertries": {
"color": {
"type": "keyword"
},
"skinType": {
"type": "keyword"
},
"species": {
"propertries": {
"terrestrial": {
"type": "keyword"
},
"aquatic": {
"type": "keyword"
}
}
},
"behavior": {
"propertries": {
"sound": {
"type": "keyword"
},
"hunt": {
"type": "keyword"
}
}
}
}
}
nota bene
In step 2 we could have a complex type such as -
{ name: "foo"
, type: [ "null", { obj1 }, { obj2 }, ... ]
, ...
}
In such a case, obj1 and obj2 may each transform into a { properties: ... } object. Using .reduce(assign, {}) means properties of obj1 will be overwritten by properties of obj2 -
: isObject(t.type)
? arr(t.type)
.map(transform)
.reduce(assign, {}) // <- cannot simply use `assign`
To remedy this, we change step 2 to merge complex types more intelligently -
: isObject(t.type)
? arr(t.type)
.map(transform)
.reduce(merge, {}) // <- define a more sophisticated merge
Where merge could be something like -
const merge = (t, u) =>
t.properties && u.properties // <- both
? { properties: Object.assign(t.properties, u.properties) }
: t.properties // <- only t
? { properties: Object.assign(t.properties, u) }
: u.properties // <- only u
? { properties: Object.assign(t, u.properties) }
: Object.assign(t, u) // <- neither
Or the same merge but using a different logical approach -
const merge = (t, u) =.
t.properties || u.properties // <- either
? { properties:
Object.assign
( t.properties || t
, u.properties || u
)
}
: Object.assign(t, u) // <- neither

I don't know your input format nor your output one. So this is probably incomplete. It captures your sample case, though, and it might serve as a baseline to which you can add clauses/conditions:
const convertField = ({name, type, fields}) =>
Array .isArray (type) && type [0] === 'null' && type [1] === 'string'
? [name, {type: 'keyword'}]
: Array .isArray (type) && type [0] === 'null' && Object (type [1]) === type [1]
? [name, {properties: Object .fromEntries (type [1] .fields .map (convertField))}]
: Object (type) === type
? [name, {properties: Object .fromEntries (type .fields .map (convertField))}]
: // else
[name, {type: 'keyword'}]
const convert = (obj) =>
convertField (obj) [1]
const input = {name: "animal", type: ["null", {type: "record", name: "zooAnimals", fields: [{name: "color", type: ["null", "string"], default: null}, {name: "skinType", type: ["null", "string"], default: null}, {name: "species", type: {type: "record", name: "AnimalSpecies", fields: [{name: "terrestrial", type: "string"}, {name: "aquatic", type: "string"}]}}, {name: "behavior", type: ["null", {type: "record", name: "AnimalBehaviors", fields: [{name: "sound", type: ["null", "string"], default: null}, {name: "hunt", type: ["null", "string"], default: null}]}], default: null}]}]}
console .log (convert (input))
.as-console-wrapper {min-height: 100% !important; top: 0}
The helper function, convertField, converts one field of your input into the format [name, <something>], where the <something> varies by the structure of the type property. In two cases we use an array of these structures as input to Object .fromEntries in order to create an object.
The main function, convert, simply grabs the second property off the result of convertField called on the root. That works if the overall structure always starts as it does in this example.
Note that the results of two of the clauses (the first and the fourth) are identical, and the other two are quite similar to one another. Also the tests for the first and second clauses are quite close. So there might be some reasonable ways to simplify this. But because the matching tests don't line up well with the matching output, it probably won't be trivial.
You can add other conditions and results easily enough. In fact, I originally wrote it with the final two lines replaced by this:
: type === 'string'
? [name, {type: 'keyword'}]
: // else
[name, {type: 'unknown'}]
which shows better where to add your other clauses, and also adds a notation (unknown) to the result if you missed a case.

How to search for a value in an array of objects that contain nested objects

I want to develop this functionality for searching/filtering a list. Basically, I'll get a search term from an input box and then I have to get all the items that include the search term from an array.
Here's what I've tried so far, it works for root level properties but doesn't work with nested arrays/objects:
// Filter List
this.filterList = query => {
if (typeof query === "string") {
// transform query to lowercase
query = query.toLowerCase();
// clear the current list being displayed
this.filteredList = [];
// filter the lsit and store the results with
// matching criteria in "filteredList" array
let filteredResults = _.filter(this.itemList, item => {
if (item && typeof item === "object") {
// loop over the item object
for (let property in item) {
if (item.hasOwnProperty(property)) {
let key = item[property];
// address, phone and emails
if (typeof key === "object" && _.isArray(key)) {
_.filter(key, element => {
if (typeof element === "object") {
for (let nestedProperty in element) {
let nestedKey = element[nestedProperty];
if (nestedKey) {
nestedKey = nestedKey.toString().toLowerCase();
}
if (nestedKey && nestedKey.includes(query)) {
return item;
}
}
}
});
} else {
if (key) key = key.toString().toLowerCase();
if (key && key.includes(query)) return item;
}
}
}
}
});
// assign the filtered results to the list being displayed
this.filteredList = [...filteredResults];
} else {
// if query is empty or null or anything other than string
// revert all changes and assign the original list to display list
this.filteredList = this.itemList;
}
};
If it helps, here's an object from the array that I am looping over:
[
{
"id": "number",
"dealerCode": "string",
"name": "string",
"gstin": "string",
"pan": "string",
"cin": "string",
"emails": [
{
"name": "string",
"address": "string",
"isPrimary": "boolean"
}
],
"phoneNumbers": [
{
"countryCode": "number",
"number": "number",
"isPrimary": "boolean"
}
],
"addresses": [
{
"addressLine1": "string",
"addressLine2": "string",
"addressLine3": "string",
"country": "string",
"state": "string",
"city": "string",
"postalCode": "number",
"isPrimary": "boolean"
}
],
"status": "string",
"statusId": "number"
}
]
I am doing this in AngularJS and using Lodash as well.

For a problem like this where you need to compare a heterogenous list of primitives and object/arrays, a recursive named function is usually the best way to go. This should probably solve what you're looking for, based on the following assumptions:
All entries by a user as treated as strings. So 99 and "99" are the same. I'll comment in the code where this assumption is made
All entries are case insensitive (all converted toLowercase)
There is no set depth of the nested objects/arrays; the solution below works recursively for any depth of a heterogeneous list
If anything matches in any leaf node, the entire object will be returned
The way the solution works below is:
Filter through the top level list and call matchesEntryInTree on each dataItem, compared to the userEntry
matchesEntryInTree will check each dataItem and see if it's an array or object
If the dataItem is an array/object, we drill into them again by calling matchesEntryInTree recursively
If it isn't, we call compareValues to see if the entry matches the current dataItem
With the recursive pattern above, all leaf nodes (regardless of the shape of the tree) will be compared to the initial userEntry
// test data for trial runs
const testData = [
{
id: 123488,
dealerCode: "ACb3",
name: "Some Name",
gstin: "string",
pan: "string",
cin: "string",
emails: [
{
name: "Some Email name",
address: "anemail.domain.com",
isPrimary: "boolean"
}
],
phoneNumbers: [
{
countryCode: "9398",
number: "number",
isPrimary: "boolean"
}
],
addresses: [
{
addressLine1: "Florida",
addressLine2: "Street place",
addressLine3: "string",
country: "string",
state: "string",
city: "string",
postalCode: "number",
isPrimary: "boolean"
}
],
status: "string",
statusId: "number"
},
{
id: 88888,
dealerCode: "NMC",
name: "Some Other",
gstin: "string",
pan: "string",
cin: "string",
emails: [
{
name: "An Email thing",
address: "athing.somewhere.org",
isPrimary: "boolean"
}
],
phoneNumbers: [
{
countryCode: "93948",
number: "number",
isPrimary: "boolean"
}
],
addresses: [
{
addressLine1: "Denver",
addressLine2: "Street place",
addressLine3: "string",
country: "string",
state: "string",
city: "string",
postalCode: "number",
isPrimary: "boolean"
}
],
status: "string",
statusId: "number"
}
];
// broke these into separate helper functions, but you can combine all of them except the recursive one if you'd like
const returnFilterResults = (userEntry, dataItems) => {
const compareValues = (entry, dataValue) => {
if ( _.isBoolean(dataValue)) {
return entry === dataValue;
} else if (_.isNumber(dataValue)) {
// if the dataValue is a number, we convert both it and the user's entry (which probably already is a string) to a string to compare
// you can make this comparison more strict if desired
return _.includes(_.toLower(_.toString(dataValue)), _.toLower(entry));
} else if (_.isString(dataValue)) {
return _.includes(_.toLower(dataValue), _.toLower(entry));
} else {
return false;
}
};
const matchesEntryInTree = (entry, dataItem) => {
// if this dataItem is an object or array, let's drill back in again
if (_.isObject(dataItem) || _.isArray(dataItem)) {
// as we recursively move through the tree, check to see if *any* of the entries match, using 'some'
return _.some(dataItem, innerDataItem => {
return matchesEntryInTree(entry, innerDataItem);
});
} else {
// if it's a primitive, then let's compare directly
return compareValues(entry, dataItem);
}
};
// I created a results variable so we could console log here in this snippet
// but you can just return from the filter directly
const results = _.filter(dataItems, dataItem => {
return matchesEntryInTree(userEntry, dataItem);
});
console.log(userEntry, results);
return results;
};
returnFilterResults("place", testData);
// both entries return
returnFilterResults("Denver", testData);
// second entry is returned
returnFilterResults(48, testData);
// both entries return - ID in first, countryCode in second
returnFilterResults(12, testData);
// first entry is returned
<script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.11/lodash.js"></script>

Why don't you use a flatten function to flatten your object/JSON then search for your value? An example for this is the following:
var flattenObject = function(ob) {
var toReturn = {};
for (var i in ob) {
if (!ob.hasOwnProperty(i)) continue;
if ((typeof ob[i]) == 'object') {
var flatObject = flattenObject(ob[i]);
for (var x in flatObject) {
if (!flatObject.hasOwnProperty(x)) continue;
toReturn[i + '.' + x] = flatObject[x];
}
} else {
toReturn[i] = ob[i];
}
}
return toReturn;
};
For a nested object, let's say
{
A : {
B: {
C: "V"
}
}
}
you'll get an object with the key A.B.C and the value "V". This way, you'll only have one level to search your value in.
Hope this helps!
Cheers!

javascript return property value from nested array of objects based on condition

i have an array of objects, in which each object could have an array of objects inside.
var mylist = [
{
"email" : null,
"school" : "schoolA",
"courses": [
{
"name" : 'ABC',
"type" : "chemistry"
},
{
"name" : 'XYZ',
"type": "math"
}
]
},
{
"email" : null,
"school": "schoolB"
}
];
i want to return course name if one of the course type is chemistry.
The course types are unique and even if they are some duplicates, we return the first one.
var result = mylist.some(function (el) {
el.courses && el.courses.some(function(u) {
if (u.type === 'chemistry') {
return u.name;
};
})
});
console.log('outcome:', result);
my code is not working at this stage.

The some callback should return a truthy or falsy value, which tells some whether to keep going (true = stop), and some returns a boolean, not a callback return value.
Probably simplest in this case just to assign directly to result:
var result;
mylist.some(function(el) {
return (el.courses || []).some(function(course) {
if (course.type === "chemistry") {
result = course.name;
return true;
}
return false;
});
});
Live Example:
var mylist = [
{
"email" : null,
"school" : "schoolA",
"courses": [
{
"name" : 'ABC',
"type" : "chemistry"
},
{
"name" : 'XYZ',
"type": "math"
}
]
},
{
"email" : null,
"school": "schoolB"
}
];
var result;
mylist.some(function(el) {
return (el.courses || []).some(function(course) {
if (course.type === "chemistry") {
result = course.name;
return true;
}
return false;
});
});
console.log(result);
I stuck to ES5 syntax since you didn't use any ES2015+ in your question, but in ES2015+, simplest probably to use nested for-of loops:
let result;
outer: for (const el of mylist) {
for (const course of el.courses || []) {
if (course.type === "chemistry") {
result = course.name;
break outer;
}
}
}
Live Example:
const mylist = [
{
"email" : null,
"school" : "schoolA",
"courses": [
{
"name" : 'ABC',
"type" : "chemistry"
},
{
"name" : 'XYZ',
"type": "math"
}
]
},
{
"email" : null,
"school": "schoolB"
}
];
let result;
outer: for (const el of mylist) {
for (const course of el.courses || []) {
if (course.type === "chemistry") {
result = course.name;
break outer;
}
}
}
console.log(result);

You could use reduce() method to iterate through each object in array and then find() method to find if some course matches type.
var mylist = [{"email":null,"school":"schoolA","courses":[{"name":"ABC","type":"chemistry"},{"name":"XYZ","type":"math"}]},{"email":null,"school":"schoolB"}]
const course = mylist.reduce((r, {courses}) => {
if (courses && !r) {
const course = courses.find(({type}) => type == 'chemistry');
if (course) r = course.name;
}
return r;
}, null)
console.log(course)

combining my objects and getting expected results

I am trying to merge my objects and get a result like below
{
"sports": {
"basketball": "kobe",
"swimming": {
},
"football": "ronaldo",
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
}
Somewhere I am doing wrong in my logic can you help me out
but if I alternate my sportA and sportsB value I am getting expected
results...not sure what problem in my current scenario
providing my code below.
fiddle
https://jsfiddle.net/tjLk0frq/3/
var sportsA ={
"sports": {
"basketball": "kobe",
"football": "ronaldo"
}
};
var sportsB ={
"sports": {
"basketball": "",
"swimming": {
},
"football": "",
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
};
function merge(sportsA, sportsB) {
for( var p in sportsB )
if( sportsA.hasOwnProperty(p) )
sportsA[p] = typeof sportsB[p] === 'object' ? merge(sportsA[p], sportsB[p]) : sportsB[p];
return sportsA;
}
merge(sportsA, sportsB );
console.log("unexpected result" + sportsA );
console.log( sportsA );
//expected
/*
{
"sports": {
"basketball": "kobe",
"swimming": {
},
"football": "ronaldo",
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
}
*/

You can use jQuery's extend method, with deep merge enabled, to do this:
var output = $.extend(true, sportsB, sportsA)
outputs:
{
"sports": {
"basketball": "kobe",
"swimming": {},
"football": "ronaldo",
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
}

You have mistake when you check sportsA.hasOwnProperty(p) in your case you only update properties that are in sportsA, but not add new from sportsB.
Also if sportsB[p] has falsy value you don't want to update it for that I've used (sportsB[p] || sportsA[p]).
Check this code.
var sportsA ={
"sports": {
"basketball": "kobe",
"football": "ronaldo"
}
};
var sportsB ={
"sports": {
"basketball": "",
"swimming": {
},
"football": "",
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
};
function merge(sportsA, sportsB) {
for( var p in sportsB )
if( sportsA.hasOwnProperty(p) ) {
sportsA[p] = typeof sportsB[p] === 'object' ? merge(sportsA[p], sportsB[p]) : (sportsB[p] || sportsA[p]);
} else {
sportsA[p] = sportsB[p];
}
return sportsA;
}
merge(sportsA, sportsB );
console.log("unexpected result" + sportsA );
console.log( sportsA );

Here you go (pure JS):
function merge(obj1, obj2) {
var result = {};
for (var prop in obj1) {
if (typeof obj1[prop] === "object" && typeof obj2[prop] === "object")
result[prop] = merge(obj1[prop], obj2[prop]);
else
result[prop] = obj1[prop];
}
for (var prop in obj2) {
result[prop] = (result[prop]? result[prop]: obj2[prop]);
}
return result;
}
console.log(merge(sportsA, sportsB));
This returns a new object, rather than modify an existing one, however.
In the first for..in loop, we check if we need to recurse first, otherwise set the property of result.
In the second for..in loop, we check if the property was already defined or if it's empty, and set the property accordingly.
Output:
{
"sports": {
"basketball": "kobe",
"football": "ronaldo",
"swimming": {},
"running": "",
"highJump": ""
},
"calendar": ["21", "25", "30"]
}
JSFiddle demo

The logic is breaking because when you only loop the property keys in one of the objects, you won't see the property keys that only exist in the other object.
You can get the root level keys of an object using Object.keys() which returns an array of the property names. Then you can merge the 2 sets of keys at same level and know all the final output properties needed
Then iterate those to get final results

JSON.stringify different from direct access to object property, strongloop

I use strongloop to build my api.
On a particular route the query includes model's relations. I get an array of objects that I would like to arrange.
In this particular arranging function I face the following problem.
The function receive an object named "item" containing a "trans" field (this field is an array of another object).
this piece of code :
console.log(JSON.stringify(item, null, 2));
produces this result :
{
"id": 1,
"created": "2015-08-19T21:04:16.000Z",
"updated": null,
"authorid": 0,
"likes": 0,
"shares": 0,
"fav": 0,
"validated": 0,
"comments": 0,
"trans": [
{
"text": "Première question en français",
"questionId": 1
}
],
"answers": [
{
"id": 1,
"questionid": 1,
"questionId": 1,
"trans": [
{
"text": "q1 : reponse 1 en francais",
"answerId": 1
}
]
},
{
"id": 2,
"questionid": 1,
"questionId": 1,
"trans": [
{
"text": "q1 : reponse 2 en francais",
"answerId": 2
}
]
}
]
}
This problem is when I try to reach this part :
item.trans[0].text
console says "item.trans is undifined" and when I try this piece of code :
console.log(item.trans);
I have this result :
function (condOrRefresh, options, cb) {
if (arguments.length === 0) {
if (typeof f.value === 'function') {
return f.value(self);
} else if (self.__cachedRelations) {
return self.__cachedRelations[name];
}
} else {
if (typeof condOrRefresh === 'function'
&& options === undefined && cb === undefined) {
// customer.orders(cb)
cb = condOrRefresh;
options = {};
condOrRefresh = undefined;
} else if (typeof options === 'function' && cb === undefined) {
// customer.orders(condOrRefresh, cb);
cb = options;
options = {};
}
options = options || {}
// Check if there is a through model
// see https://github.com/strongloop/loopback/issues/1076
if (f._scope.collect &&
condOrRefresh !== null && typeof condOrRefresh === 'object') {
//extract the paging filters to the through model
['limit','offset','skip','order'].forEach(function(pagerFilter){
if(typeof(condOrRefresh[pagerFilter]) !== 'undefined'){
f._scope[pagerFilter] = condOrRefresh[pagerFilter];
delete condOrRefresh[pagerFilter];
}
});
// Adjust the include so that the condition will be applied to
// the target model
f._scope.include = {
relation: f._scope.collect,
scope: condOrRefresh
};
condOrRefresh = {};
}
return definition.related(self, f._scope, condOrRefresh, options, cb);
}
}
How can I simply access the "trans" property in this case to get the text inside ?
(Not really at easy in js)
Thanks in advance.

It's possible that your item object has implemented the toJSON function.
Pop open your browser's console and run this snippet to see an example of how this can make a difference between the stringified JSON and the actual object:
var x = {
name: "foo",
children : function() {
return [ { name: 'child 1' }, { name: 'child 2' } ];
},
toJSON: function() {
var simplified = { name: this.name, children: this.children() };
return simplified
}
};
// shows children as a simple array
console.log( JSON.stringify( x, null, 2 ) );
// {
// "name": "foo",
// "children": [
// {
// "name": "child 1"
// },
// {
// "name": "child 2"
// }
// ]
// }
// oops... not what you expected
console.log( x.children[0].name );
// Uncaught TypeError: Cannot read property 'name' of undefined
Of course, the easiest fix would be to parse the stringify result:
var y = JSON.parse( JSON.stringify( x ) );
console.log( y.children[0].name );
It's a last-case-scenario-type solution, though, since JSON.stringify is a very expensive function.