Count duplicates within an Array of Objects - javascript

I have an array of objects as follows within my server side JS:
[
{
"Company": "IBM"
},
{
"Person": "ACORD LOMA"
},
{
"Company": "IBM"
},
{
"Company": "MSFT"
},
{
"Place": "New York"
}
]
I need to iterate through this structure, detect any duplicates and then create a count of a duplicate is found along side each value.
Both of the values must match to qualify as a duplicate e.g. "Company": "IBM" is not a match for "Company": "MSFT".
I have the options of changing the inbound array of objects if needed. I would like the output to be an object, but am really struggling to get this to work.
EDIT: Here is the code I have so far where processArray is the array as listed above.
var returnObj = {};
for(var x=0; x < processArray.length; x++){
//Check if we already have the array item as a key in the return obj
returnObj[processArray[x]] = returnObj[processArray[x]] || processArray[x].toString();
// Setup the count field
returnObj[processArray[x]].count = returnObj[processArray[x]].count || 1;
// Increment the count
returnObj[processArray[x]].count = returnObj[processArray[x]].count + 1;
}
console.log('====================' + JSON.stringify(returnObj));

For example:
counter = {}
yourArray.forEach(function(obj) {
var key = JSON.stringify(obj)
counter[key] = (counter[key] || 0) + 1
})
Docs: Array.forEach, JSON.stringify.

Object.prototype.equals = function(o){
for(var key in o)
if(o.hasOwnProperty(key) && this.hasOwnProperty(key))
if(this[key] != o[key])
return false;
return true;
}
var array = [/*initial array*/],
newArray = [],
ok = true;
for(var i=0,l=array.length-1;i<l;i++)
for(var j=i;j<l+1;j++)
{
if(!array[i].equals(array[j]))
newArray.push(array[i]);
}

We are required to write a JavaScript function that takes in one such array of objects. The function creates and return a new array in which no objects are repeated (by repeated we mean objects having same value for "Country" property.)
Moreover, the function should assign a count property to each object that represents the number of times they appeared in the original array.
const arr = [
{
"Country": "BR",
"New Lv1−Lv2": "#N/A"
},
{
"Country": "BR",
"New Lv1−Lv2": "#N/A"
},
{
"Country": "",
"New Lv1−Lv2": "test"
}];
const convert = (arr) => {
const res = {};
arr.forEach((obj) => {
const key = `${obj.Country}${obj["New Lv1−Lv2"]}`;
if (!res[key]) {
res[key] = { ...obj, count: 0 };
};
res[key].count += 1;
});
return Object.values(res);
};
console.log(convert(arr));
know more

With ES6, one can use Array#reduce with an object to store the counts.
let counts = arr.reduce((acc, curr)=>{
const str = JSON.stringify(curr);
acc[str] = (acc[str] || 0) + 1;
return acc;
}, {});
Demo
To create a new array without duplicates, a Set can be used with Array#filter.
let set = new Set;
let res = arr.filter(x => {
const str = JSON.stringify(x);
return !set.has(str) && set.add(str);
});
Demo

Related

Cleaning the json object by removing duplicates and null and merging them into a single record

Cleaning the JSON object by removing duplicates and null and merging them into a single record
The json array looks like this:
var result =
[
{"id":"10035","occupation":null,"state":"FL"},
{"id":"10035","occupation":"doctor","state":null},
{"id":"10035","occupation":null,"state":null},
]
I want to merge records into one neglecting all the null fields and make it as a single record.Below is my expected output:
[
{"id":"10035","occupation":"doctor","state":"FL"}
]
You could do it with this ES6 script:
let data = [
{"id":"10035","occupation":null,"state":"FL"},
{"id":"10035","occupation":"doctor","state":null},
{"id":"10035","occupation":null,"state":null},
];
let result = Object.values(data.reduce ( (acc, {id, occupation, state}) => {
acc[id] = Object.assign({ id }, acc[id],
occupation && { occupation },
state && { state });
return acc;
}, {}));
console.log(result);
It will still produce multiple records if you have different id values in your input. When there are more than one non-null values for the other properties, but for the same id, then only the last one will survive.
When you're without support for Object.values
Use this definition of it:
Object.values = Object.values || (o => Object.keys(o).map(k => o[k]));
var final = {};
for (var i in result) {
for (var k in result[i]) {
if (result[i][k] && final[k] !== result[i][k]) {
final[k] = result[i][k];
}
}
}
console.log(final); // outputs: {id: "10035", state: "FL", occupation: "doctor"}
Here's a simple to understand example, which works for objects with any number of properties.
let data = [
{"id":"10035","occupation":null,"state":"FL"},
{"id":"10035","occupation":"doctor","state":null},
{"id":"10035","occupation":null,"state":null},
];
let result = data[0];
data.forEach(obj=> { // iterate through all objects in array
for(key in obj) // iterate through all properties of objects
if(obj[key]) result[key] = obj[key]; // if not null, assign to final result
});
console.log(result);
Here is a way to do it in O(n) time:
const mergeObjects = (data) => {
const dataWithoutDuplicates = {};
// first pass will get rid of dupes
let user;
for(let i = 0; i < data.length; data++) {
user = data[i];
if(!dataWithoutDuplicates[user.id]) {
dataWithoutDuplicates[user.id] = user
} else {
Object.keys(dataWithoutDuplicates[user.id]).forEach(key => {
if(dataWithoutDuplicates[user.id][key] === null && user[key]) {
dataWithoutDuplicates[user.id][key] = user[key]
}
})
}
return Object.values(dataWithoutDuplicates)
}

Filtering JSON Array

I need to add filter option to my grid.I use Fixed Data Table.Here is simple filtering example with that grid.
https://github.com/facebook/fixed-data-table/blob/master/examples/old/FilterExample.js
This example filter the Json array only by first name.But I need to filter by all of the objects in JSON Array.
For example may JSON array is here:
{"id":7,"first_name":"Sarah","last_name":"Hottie",
"country":"Sweden","salary":12000},
{"id":9,"first_name":"Mary","last_name":"Parah",
"country":"Argentina","salary":10000}
When I write "arah" to the general input filter value.I need to show both of the two elements of array.Because "id:7" first name (Sarah) and "id:9" last name (Parah) include my filter value ("arah").
If the country value of the another element of JSON array include "arah" I need to show that too.
So I need to filter the JSON array by all of the values it include.
What do you suggest?
You can utilize the filter prototype of the array. It will be something like this:
var arr = [ {"id":7,"first_name":"Sarah","last_name":"Hottie",
"country":"Sweden","salary":12000}, {"id":9,"first_name":"Mary","last_name":"Parah","country":"Argentina","salary":10000}]
var runFilter = function(arr,searchKey) {
var filterFn = function(obj) {
// Iterate the obj for each key.
for (var k in obj) {
if (typeof obj[k] == "string" && obj[k].indexOf(searchKey) >= 0) {
return true;
}
}
}
return arr.filter(filterFn);
}
var filteredArr = runFilter(arr,'arah')
I suggest to use Array#filter in combination with Array#some and a check of the type.
var data = [{ "id": 7, "first_name": "Sarah", "last_name": "Hottie", "country": "Sweden", "salary": 12000 }, { "id": 9, "first_name": "Mary", "last_name": "Parah", "country": "Argentina", "salary": 10000 }],
search = 'arah',
result = data.filter(function (a) {
return Object.keys(a).some(function (k) {
if (typeof a[k] === 'string' && ~a[k].indexOf(search)) {
return true;
}
if (typeof a[k] === 'number' && ~a[k] === search) {
return true;
}
});
});
document.write('<pre>' + JSON.stringify(result, 0, 4) + '</pre>');
You can find the filter function in line 45 of the example code. It is
return row['firstName'].toLowerCase().indexOf(filterBy.toLowerCase()) >= 0
If you want to look into every part of an Object, you can use a for...in loop:
for(var key in row){
if((row[key] + "").indexOf(filterBy) > -1){
return true;
}
}
return false;
Replace line 45 with the code above and you should be fine.
Try This :
<script type="text/javascript">
var arr = [ {"id":7,"first_name":"Sarah","last_name":"Hottie","country":"Sweden","salary":12000},
{"id":8,"first_name":"Mary","last_name":"Parah","country":"Argentina","salary":10000},
{"id":9,"first_name":"Gold","last_name":"sonam","country":"India","salary":15000}];
var filterKey = 'arah';
function findJsonString(arr,filterKey){
var result = [];
for (var i = arr.length - 1; i >= 0; i--) {
var part1 = arr[i].first_name.indexOf(filterKey);
var part2 = arr[i].last_name.indexOf(filterKey);
// console.log(arr[i]);
// console.log(' part 1 : ' + part1 + ' part 2 : ' + part2);
if(part1 != -1 || part2 != -1)
{
result[+i] = arr[i];
// OR result.push(arr[i]);
}
}
return result;
}
console.log(findJsonString(arr,filterKey));
</script>
OUTPUT :
[Object { id=7, first_name="Sarah", last_name="Hottie", more...}, Object { id=8, first_name="Mary", last_name="Parah", more...}]

How to convert array of key–value objects to array of objects with a single property?

I have an array of objects like this:
[
{ "key": "fruit", "value": "apple" },
{ "key": "color", "value": "red" },
{ "key": "location", "value": "garden" }
]
I need to convert it to the following format:
[
{ "fruit": "apple" },
{ "color": "red" },
{ "location": "garden" }
]
How can this be done using JavaScript?
You can use .map
var data = [
{"key":"fruit","value":"apple"},
{"key":"color","value":"red"},
{"key":"location","value":"garden"}
];
var result = data.map(function (e) {
var element = {};
element[e.key] = e.value;
return element;
});
console.log(result);
also if you use ES2015 you can do it like this
var result = data.map((e) => {
return {[e.key]: e.value};
});
Example
Using an arrow function, with the data called arr
arr.map(e => {
var o = {};
o[e.key] = e.value;
return o;
});
This generates a new Array and does not modify the original
It can be simplified down to one line as
arr.map(e => ({[e.key]: e.value}));
If you can't assume arrow function support yet, you would write this longhand
arr.map(function (e) {
var o = {};
o[e.key] = e.value;
return o;
});
Using map (as suggested in other answers) or the following will do what you want...
var data = [{"key":"fruit","value":"apple"},{"key":"color","value":"red"},{"key":"location","value":"garden"}];
var obj = {};
for(var i = 0; i < data.length; i++) {
obj[data[i]["key"]] = data[i]["value"];
}
In Javascript, obj.property and obj['property'] return same things.
obj['property'] is more flexible because the key 'property' could be a string with some space :
obj['pro per ty'] // work
obj.pro per ty // not work
or
var a = 'property';
obj.a == obj.property // => false
obj[a] == obj.property // => true
So you could try that.
var data = [{"key":"fruit","value":"apple"},{"key":"color","value":"red"},{"key":"location","value":"garden"}]
var new_data = [];
var data_length = data.length; // just a little optimisation for-loop
for (var i = 0; i < data_length; i++) {
var item = data[i]; // to have a vision close of foreach-loop (foreach item of collection)
new_data[i] = {};
new_data[i][item.key] = item.value;
}
console.log(new_data);
// [{"fruit":"apple"},{"color":"red"},{"location":"garden"}]
What you currently have is an array of object, each having two attributes, key and value. If you are not aware of map, you can always run a forEach loop on this array and rearrange the data. Try something like below:
function() {
var newArray = [];
oldArray.forEach(function(x){
var obj= {};
obj[x.key] = x.value;
newArray.push(obj);
});
console.log(newArray);
}
here oldArray is your original data

Remove duplicate objects from an array using javascript

I am trying to figure out an efficient way to remove objects that are duplicates from an array and looking for the most efficient answer. I looked around the internet everything seems to be using primitive data... or not scalable for large arrays. This is my current implementation which is can be improved and want to try to avoid labels.
Test.prototype.unique = function (arr, artist, title, cb) {
console.log(arr.length);
var n, y, x, i, r;
r = [];
o: for (i = 0, n = arr.length; i < n; i++) {
for (x = 0, y = r.length; x < y; x++) {
if (r[x].artist == arr[i].artist && r[x].title == arr[i].title) {
continue o;
}
}
r.push(arr[i]);
}
cb(r);
};
and the array looks something like this:
[{title: sky, artist: jon}, {title: rain, artist: Paul}, ....]
Order does not matter, but if sorting makes it more efficient then I am up for the challenge...
and for people who do not know o is a label and it is just saying jump back to the loop instead of pushing to the new array.
Pure javascript please no libs.
ANSWERS SO FAR:
The Performance Test for the answers below:
http://jsperf.com/remove-duplicates-for-loops
I see, the problem there is that the complexity is squared. There is one trick to do it, it's simply by using "Associative arrays".
You can get the array, loop over it, and add the value of the array as a key to the associative array. Since it doesn't allow duplicated keys, you will automatically get rid of the duplicates.
Since you are looking for title and artist when comparing, you can actually try to use something like:
var arrResult = {};
for (i = 0, n = arr.length; i < n; i++) {
var item = arr[i];
arrResult[ item.title + " - " + item.artist ] = item;
}
Then you just loop the arrResult again, and recreate the array.
var i = 0;
var nonDuplicatedArray = [];
for(var item in arrResult) {
nonDuplicatedArray[i++] = arrResult[item];
}
Updated to include Paul's comment. Thanks!
Here is a solution that works for me.
Helper functions:
// sorts an array of objects according to one field
// call like this: sortObjArray(myArray, "name" );
// it will modify the input array
sortObjArray = function(arr, field) {
arr.sort(
function compare(a,b) {
if (a[field] < b[field])
return -1;
if (a[field] > b[field])
return 1;
return 0;
}
);
}
// call like this: uniqueDishes = removeDuplicatesFromObjArray(dishes, "dishName");
// it will NOT modify the input array
// input array MUST be sorted by the same field (asc or desc doesn't matter)
removeDuplicatesFromObjArray = function(arr, field) {
var u = [];
arr.reduce(function (a, b) {
if (a[field] !== b[field]) u.push(b);
return b;
}, []);
return u;
}
and then simply call:
sortObjArray(dishes, "name");
dishes = removeDuplicatesFromObjArray(dishes, "name");
Basic sort-then-unique implementation, fiddle HERE:
function unique(arr) {
var comparer = function compareObject(a, b) {
if (a.title == b.title) {
if (a.artist < b.artist) {
return -1;
} else if (a.artist > b.artist) {
return 1;
} else {
return 0;
}
} else {
if (a.title < b.title) {
return -1;
} else {
return 1;
}
}
}
arr.sort(comparer);
console.log("Sorted: " + JSON.stringify(arr));
for (var i = 0; i < arr.length - 1; ++i) {
if (comparer(arr[i], arr[i+1]) === 0) {
arr.splice(i, 1);
console.log("Splicing: " + JSON.stringify(arr));
}
}
return arr;
}
It may or may not be the most efficient, and should be entirely scalable. I've added some console.logs so you can see it as it works.
EDIT
In the interest of saving on the space the function used, I did that for loop at the end, but it seems likely that didn't properly find only unique results (depsite it passing my simple jsfiddle test). Please try replacing my for loop with the following:
var checker;
var uniqueResults = [];
for (var i = 0; i < arr.length; ++i) {
if (!checker || comparer(checker, arr[i]) != 0) {
checker = arr[i];
uniqueResults.push(checker);
}
}
return uniqueResults;
I use this function. its not doing any sorting, but produces result. Cant say about performance as never measure it.
var unique = function(a){
var seen = [], result = [];
for(var len = a.length, i = len-1; i >= 0; i--){
if(!seen[a[i]]){
seen[a[i]] = true;
result.push(a[i]);
}
}
return result;
}
var ar = [1,2,3,1,1,1,1,1,"", "","","", "a", "b"];
console.log(unique(ar));// this will produce [1,2,3,"", "a", "b"] all unique elements.
Below is Henrique Feijo's answer with ample explanation and an example that you can cut and paste:
Goal: Convert an array of objects that contains duplicate objects (like this one)...
[
{
"id": 10620,
"name": "Things to Print"
},
{
"id": 10620,
"name": "Things to Print"
},
{
"id": 4334,
"name": "Interesting"
}
]
... Into an array of objects without duplicate objects (like this one):
[
{
"id": 10620,
"name": "Things to Print"
},
{
"id": 4334,
"name": "Interesting"
}
]
Explanation provided in the comments:
var allContent = [{
"id": 10620,
"name": "Things to Print"
}, {
"id": 10620,
"name": "Things to Print"
}, {
"id": 4334,
"name": "Interesting"
}]
//Put Objects Into As Associative Array. Each key consists of a composite value generated by each set of values from the objects in allContent.
var noDupeObj = {} //Create an associative array. It will not accept duplicate keys.
for (i = 0, n = allContent.length; i < n; i++) {
var item = allContent[i]; //Store each object as a variable. This helps with clarity in the next line.
noDupeObj[item.id + "|" + item.name] = item; //This is the critical step.
//Here, you create an object within the associative array that has a key composed of the two values from the original object.
// Use a delimiter to not have foo+bar handled like fo+obar
//Since the associative array will not allow duplicate keys, and the keys are determined by the content, then all duplicate content are removed.
//The value assigned to each key is the original object which is along for the ride and used to reconstruct the list in the next step.
}
//Recontructs the list with only the unique objects left in the doDupeObj associative array
var i = 0;
var nonDuplicatedArray = [];
for (var item in noDupeObj) {
nonDuplicatedArray[i++] = noDupeObj[item]; //Populate the array with the values from the noDupeObj.
}
console.log(nonDuplicatedArray)
For those who love ES6 and short stuff, here it's one solution:
const arr = [
{ title: "sky", artist: "Jon" },
{ title: "rain", artist: "Paul" },
{ title: "sky", artist: "Jon" }
];
Array.from(arr.reduce((a, o) => a.set(o.title, o), new Map()).values());
const arr = [
{ title: "sky", artist: "Jon" },
{ title: "rain", artist: "Paul" },
{ title: "sky", artist: "Jon" },
{ title: "rain", artist: "Jon" },
{ title: "cry", artist: "Jon" }
];
const unique = Array.from(arr.reduce((a, o) => a.set(o.title, o), new Map()).values());
console.log(`New array length: ${unique.length}`)
console.log(unique)
The above example only works for a unique title or id. Basically, it creates a new map for songs with duplicate titles.
Below code compares object with JSON as String format and removes duplicates and works fine with simple arrays.
Array.prototype.unique=function(a){
return function(){
return this.filter(a)
}
}(
function(a,b,c){
var tmp=[];
c.forEach(function(el){
tmp.push(JSON.stringify(el))
});
return tmp.indexOf(JSON.stringify(a),b+1)<0
})
If you are using underscore js, it is easy to remove duplicate object.
http://underscorejs.org/#uniq
function remove_duplicates(objectsArray) {
var arr = [], collection = [];
$.each(objectsArray, function (index, value) {
if ($.inArray(value.id, arr) == -1) {
arr.push(value.id);
collection.push(value);
}
});
return collection;
}

Remove duplicate objects from array by a certain merge algorithm

Here's what I have in mind:
Given an array of objects:
[
{
"name": "Kirk",
"count": 1
},
{
"name": "Spock",
"count": 1
},
{
"name": "Kirk",
"count": 1
}
]
I am trying to get:
[
{
"name": "Kirk",
"count": 2
},
{
"name": "Spock",
"count": 1
}
]
I am wondering if there's already an algorithm, perhaps combining some higher order functions to achieve this. I could do this easily with loops, but I am looking for a way to solve it using higher order functions. If someone could point me to what I should use to achieve this, it would be great. Again, I'm looking for something as elegant as possible (two maps and a filter would not be a big improvement from loops).
This is my current solution and I'm looking for something better (and by better I mean more expressive):
function mergeDuplicates(input) {
var output = [];
var existingItem = null;
input.forEach(function (inputItem) {
existingItem = _.find(output, function (outputItem) {
return inputItem.name === outputItem.name;
});
existingItem ? existingItem.count += 1 : output.push({
name: inputItem.name,
count: 1
});
existingItem = null;
});
return output;
}
To make line #10 more clear: in the original array, count might be either non-existing or 1, hence I set it to 1.
I think the best way would be to hash each object if it does not already exist, and delete the ones that you found already hashed in your structure. This way, you'd be checking the existence of each object only 1 (depends on your hash scheme).
Just a function if you would like to use.
function merge(arr) {
for(var o = {}, i; i=arr.shift(); o[i.name] = i.count + (o[i.name] || 0));
for(i in o) arr.push({name:i, count:o[i]});
}
Calling :
var myArray = [{"name":"Kirk","count":1},
{"name":"Spock","count":1},
{"name":"Kirk","count":1}];
merge(myArray);
// myArray is now : [{"name":"Kirk","count":2}, {"name":"Spock","count":1}]
You can use reduce which is actually a fold.
a.reduce(function(p, c) {
var n = c.name;
if (p[n])
p[n].count++;
else
p[n] = c;
return p;
}, {})
will give you a object with "Kirk" and "Spock" as the key, what you want as values.
I know this is an old question, but I couldn't resist trying to solve it. Instead of two maps and a filter, we use a sort and then a reduce. This was a fun one to sort out :-)
function mergeDuplicates(list, prop, cb){
return list.sort(function(a,b){
if(a[prop] < b[prop]){ return -1;}
if(a[prop] > b[prop]){return 1;}
return 0;
}).reduce(function(acc, item, index, array){
if(index > 0 && array[index-1][prop] === item[prop]){
cb(acc[acc.length-1], item);
return acc;
}else{
var newItem = Object.assign({}, item);
cb(newItem);
acc.push(newItem);
return acc;
}
}, []);
}
Then use it like this:
var newList = mergeDuplicates(list, "name", function(item, dup){
if(dup){
item.count++;
}else{
item.count = 1;
}
});
EDIT: Here's another take at it using reduce and using an object as a hashmap to store duplicates (similar to some of the other answers). This one uses ramdajs
const mergeDups = (cb, prop, list) => R.pipe(
R.reduce((acc, item) => (
R.has(item[prop], acc) ?
R.assoc(item[prop], cb(acc[item[prop]], item), acc) :
R.assoc(item[prop], cb(item), acc)
), {}),
R.values
)(list);
const cb = (i, d) => ( !R.isNil(d) ?
R.assoc('count', i.count + 1, i) :
R.assoc('count', 1, i) )
mergeDups(cb, 'name', items);
Here it is in the repl on Ramda's site
Try this better i'll useful resolve your issues
cleanup(arrayOfObj, 'name');
function cleanup(arr, prop) {
var new_arr = [];
var lookup = {};
for (var i in arr) {
lookup[arr[i][prop]] = arr[i];
}
for (i in lookup) {
new_arr.push(lookup[i]);
}
return new_arr;
}
Yet another version using reduce function:
var items =
[
{
"name": "Kirk",
"count": 1
},
{
"name": "Spock",
"count": 1
},
{
"name": "Kirk",
"count": 1
}
];
var filtered = items.reduce(function(prev, current,index){
if(!(current.name in prev.keys)) {
prev.keys[current.name] = index;
prev.result.push(current);
}
else{
prev.result[prev.keys[current.name]].count += current.count;
}
return prev;
},{result: [], keys: []}).result;
document.getElementById("output").innerHTML = JSON.stringify(filtered,null,2);
<pre id='output' />

Categories