We Keep Coding

How to highlight duplicates faster in google sheets script?

I wrote function markDupes1Col which highlights all duplicates in column. It works great, but when number of rows has passed 50k it became really slow. Is there anything I can do to make it faster?
function removeEmptyCells(values) {
values = values.filter(function (el) {
return el != null && el[0] !== '' && el[0] != null;
});
return values;
}
function findDupes(arr) {
var sortedData = arr.slice().sort();
var duplicates = [];
for (var i = 0; i < sortedData.length; i++) {
if (sortedData[i] && sortedData[i] !== '' && sortedData[i + 1] == sortedData[i]) {
duplicates.push(sortedData[i]);
}
}
return duplicates;
}
function markDupes1Col() {
var ss = SpreadsheetApp.openById(appId);
var sheetName = arguments[0];
var sheet = ss.getSheetByName(sheetName);
for(var n = 1; n < arguments.length; n++) {
var lastRow = sheet.getLastRow();
if (lastRow == 0) lastRow = 1;
var rangeArray = sheet.getRange(1, arguments[n], lastRow);
var valuesArray = rangeArray.getValues();
valuesArray = removeEmptyCells(valuesArray);
// Convert to one dimensional array
valuesArray = [].concat.apply([], valuesArray);
var duplicates = findDupes(valuesArray);
rangeArray.setBackground(null);
if (duplicates.length > 0) {
for (var i = 0; i < valuesArray.length; i++) {
for (var j = 0; j < duplicates.length; j++) {
if (valuesArray[i] == duplicates[j]) {
sheet.getRange(i + 1, arguments[n]).setBackground("#b7e1cd");
break;
}
}
}
}
}
}

Issue(Slow performance):
Use of setBackground in a loop for each cell.
Use of arrays to store duplicates.
Solution:
Create a output array and use setBackgrounds() instead.
Use Objects {} to store duplicates
If the above solutions are still slow, use sheets api to batch set backgrounds
Snippet:
function findDupes(arr){
var valObj = {};
var duplicates = {};
arr.forEach(function(row){
var el = row[0];
if(el in valObj){
duplicates[el] = 1
} else {
valObj[el] = 1;
}
})
return duplicates;
}
//....
valuesArray = removeEmptyCells(valuesArray);
//valuesArray = [].concat.apply([], valuesArray); Removed
var duplicates = findDupes(valuesArray);
//.....
rangeArray.setBackgrounds(
valuesArray.map(function(row){
return [(row[0] in duplicates) ? "#b7e1cd" : null]
})
)
References:
Best practices § Use Batch Operations
Range § setBackgrounds

show most frequently occuring input value [duplicate]

var store = ['1','2','2','3','4'];
I want to find out that 2 appear the most in the array. How do I go about doing that?

I would do something like:
var store = ['1','2','2','3','4'];
var frequency = {}; // array of frequency.
var max = 0; // holds the max frequency.
var result; // holds the max frequency element.
for(var v in store) {
frequency[store[v]]=(frequency[store[v]] || 0)+1; // increment frequency.
if(frequency[store[v]] > max) { // is this frequency > max so far ?
max = frequency[store[v]]; // update max.
result = store[v]; // update result.
}
}

Solution with emphasis to Array.prototype.forEach and the problem of getting more than one key if the max count is shared among more items.
Edit: Proposal with one loop, only.
var store = ['1', '2', '2', '3', '4', '5', '5'],
distribution = {},
max = 0,
result = [];
store.forEach(function (a) {
distribution[a] = (distribution[a] || 0) + 1;
if (distribution[a] > max) {
max = distribution[a];
result = [a];
return;
}
if (distribution[a] === max) {
result.push(a);
}
});
console.log('max: ' + max);
console.log('key/s with max count: ' + JSON.stringify(result));
console.log(distribution);

arr.sort();
var max=0,result,freq = 0;
for(var i=0; i < arr.length; i++){
if(arr[i]===arr[i+1]){
freq++;
}
else {
freq=0;
}
if(freq>max){
result = arr[i];
max = freq;
}
}
return result;

Make a histogram, find the key for the maximum number in the histogram.
var hist = [];
for (var i = 0; i < store.length; i++) {
var n = store[i];
if (hist[n] === undefined) hist[n] = 0;
else hist[n]++;
}
var best_count = hist[store[0]];
var best = store[0];
for (var i = 0; i < store.length; i++) {
if (hist[store[i]] > best_count) {
best_count = hist[store[i]];
best = store[i];
}
}
alert(best + ' occurs the most at ' + best_count + ' occurrences');
This assumes either there are no ties, or you don't care which is selected.

Another ES6 option. Works with strings or numbers.
function mode(arr) {
const store = {}
arr.forEach((num) => store[num] ? store[num] += 1 : store[num] = 1)
return Object.keys(store).sort((a, b) => store[b] - store[a])[0]
}

If the array is sorted this should work:
function popular(array) {
if (array.length == 0) return [null, 0];
var n = max = 1, maxNum = array[0], pv, cv;
for(var i = 0; i < array.length; i++, pv = array[i-1], cv = array[i]) {
if (pv == cv) {
if (++n >= max) {
max = n; maxNum = cv;
}
} else n = 1;
}
return [maxNum, max];
};
popular([1,2,2,3,4,9,9,9,9,1,1])
[9, 4]
popular([1,2,2,3,4,9,9,9,9,1,1,10,10,10,10,10])
[10, 5]

This version will quit looking when the count exceeds the number of items not yet counted.
It works without sorting the array.
Array.prototype.most= function(){
var L= this.length, freq= [], unique= [],
tem, max= 1, index, count;
while(L>= max){
tem= this[--L];
if(unique.indexOf(tem)== -1){
unique.push(tem);
index= -1, count= 0;
while((index= this.indexOf(tem, index+1))!= -1){
++count;
}
if(count> max){
freq= [tem];
max= count;
}
else if(count== max) freq.push(tem);
}
}
return [freq, max];
}
//test
var A= ["apples","oranges","oranges","oranges","bananas",
"bananas","oranges","bananas"];
alert(A.most()) // [oranges,4]
A.push("bananas");
alert(A.most()) // [bananas,oranges,4]

I solved it this way for finding the most common integer
function mostCommon(arr) {
// finds the first most common integer, doesn't account for 2 equally common integers (a tie)
freq = [];
// set all frequency counts to 0
for(i = 0; i < arr[arr.length-1]; i++) {
freq[i] = 0;
}
// use index in freq to represent the number, and the value at the index represent the frequency count
for(i = 0; i < arr.length; i++) {
freq[arr[i]]++;
}
// find biggest number's index, that's the most frequent integer
mostCommon = freq[0];
for(i = 0; i < freq.length; i++) {
if(freq[i] > mostCommon) {
mostCommon = i;
}
}
return mostCommon;
}

This is my solution.
var max_frequent_elements = function(arr){
var a = [], b = [], prev;
arr.sort();
for ( var i = 0; i < arr.length; i++ ) {
if ( arr[i] !== prev ) {
a.push(arr[i]);
b.push(1);
} else {
b[b.length-1]++;
}
prev = arr[i];
}
var max = b[0]
for(var p=1;p<b.length;p++){
if(b[p]>max)max=b[p]
}
var indices = []
for(var q=0;q<a.length;q++){
if(b[q]==max){indices.push(a[q])}
}
return indices;
};

All the solutions above are iterative.
Here's a ES6 functional mutation-less version:
Array.prototype.mostRepresented = function() {
const indexedElements = this.reduce((result, element) => {
return result.map(el => {
return {
value: el.value,
count: el.count + (el.value === element ? 1 : 0),
};
}).concat(result.some(el => el.value === element) ? [] : {value: element, count: 1});
}, []);
return (indexedElements.slice(1).reduce(
(result, indexedElement) => (indexedElement.count > result.count ? indexedElement : result),
indexedElements[0]) || {}).value;
};
It could be optimized in specific situations where performance is the bottleneck, but it has a great advantage of working with any kind of array elements.
The last line could be replaced with:
return (indexedElements.maxBy(el => el.count) || {}).value;
With:
Array.prototype.maxBy = function(fn) {
return this.slice(1).reduce((result, element) => (fn(element) > fn(result) ? element : result), this[0]);
};
for clarity

If the array contains strings try this solution
function GetMaxFrequency (array) {
var store = array;
var frequency = []; // array of frequency.
var result; // holds the max frequency element.
for(var v in store) {
var target = store[v];
var numOccurences = $.grep(store, function (elem) {
return elem === target;
}).length;
frequency.push(numOccurences);
}
maxValue = Math.max.apply(this, frequency);
result = store[$.inArray(maxValue,frequency)];
return result;
}
var store = ['ff','cc','cc','ff','ff','ff','ff','ff','ff','yahya','yahya','cc','yahya'];
alert(GetMaxFrequency(store));

A fairly short solution.
function mostCommon(list) {
var keyCounts = {};
var topCount = 0;
var topKey = {};
list.forEach(function(item, val) {
keyCounts[item] = keyCounts[item] + 1 || 1;
if (keyCounts[item] > topCount) {
topKey = item;
topCount = keyCounts[item];
}
});
return topKey;
}
document.write(mostCommon(['AA', 'AA', 'AB', 'AC']))

This solution returns an array of the most appearing numbers in an array, in case multiple numbers appear at the "max" times.
function mode(numbers) {
var counterObj = {};
var max = 0;
var result = [];
for(let num in numbers) {
counterObj[numbers[num]] = (counterObj[numbers[num]] || 0) + 1;
if(counterObj[numbers[num]] >= max) {
max = counterObj[numbers[num]];
}
}
for (let num in counterObj) {
if(counterObj[num] == max) {
result.push(parseInt(num));
}
}
return result;
}

Javascript, consolidate array elements

I'm stuck of finding a way to consolidate array elements.
so my array is in format of [id1:port1,id2:port2,id1:port3,id2:port4,id5:port5...] where each element has 2 portions. The id portion is not unique. what I try to consolidate is to create a new array will have data like [id1#port1:port3,id2#port2:port4,id5#port5]
I tried code below but it didn't get me too far. can any guru help me out?
var orinString = "id1:port1,id2:port2,id1:port3,id2:port4,id5:port5";
var newArray1 = orinString.split(",");
var newArray2 = orinString.split(",");
var newArray3 = [];
for (x=0; x<=newArray1.length-1; x++) {
for (y=0; y<= newArray2.length-1; y++) {
if ((newArray1[x].split(":")[0] == newArray2[y].split(":")[0]) && (newArray1[x].split(":")[1] != newArray2[y].split(":")[1])) {
newArray3.push(newArray1[x].split(":")[0] +"#"+ newArray1[x].split(":")[1]);
}
}
}
for (z=0; z<=newArray3.length; z++) {
gs.log("show me the result " +newArray3[z]);
}

is it that you want:
var orinString = "id1:port1,id2:port2,id1:port3,id2:port4,id5:port5";
var arr1 = orinString.split(",");
var temp= "";
var newStr = "";
arr1.sort();
for(i=0; i< arr1.length; i++) {
var item = arr1[i].split(':');
if(item[0] !== temp || temp === "") {
newStr += "," + item[0] + "#" + item[1];
} else {
newStr += ":"+item[1];
}
temp = item[0];
}
console.log(newStr.substring(1));

A typical way to solve a problem like this is
Convert them into workable values
Populate some kind of lookup table
Output the results of this lookup table
For example
var orinString = "id1:port1,id2:port2,id1:port3,id2:port4,id5:port5";
var idsAndPorts = orinString.split(",");
// Populate a key lookup
var hashTable = {};
idsAndPorts.forEach(function(s) {
var splitValue = s.split(':');
var key = splitValue[0];
var value = splitValue[1];
if(hashTable[key]) {
hashTable[key].push(value);
} else {
hashTable[key] = [value];
}
});
// Now convert it back into an array again
var finalArray = [];
for(var k in hashTable) {
finalArray.push(k + '#' + hashTable[k].join(','));
}
// View the results
finalArray.forEach(function(f) {
console.log(f);
})
This does not guarantee the final array will be sorted, but you can sort it yourself if you wish.

removing multiple arrays from a 2D array

My web app is taking in arbitrarily large 2D arrays that sometimes look something like this:
var multiArray = [["","","",""],[1,2,3],["hello","dog","cat"],["","","",""]];
I want to write a function to take out every array inside of multiArray that is comprised entirely of quotes. In other words, any array that looks like this:
["","","",""]
should be deleted from multiArray.
I tried writing the following function, but the problem with using splice in a for loop is that splicing will change the length of the array, and I end up trying to access undefined elements. Please help!
Thanks!
Here's the incorrect function I wrote:
function cleanWhitespace(arrayOfArrays) {
var i;
var arrayOfArraysLength = arrayOfArrays.length;
for (i = 0; i < arrayOfArraysLength; i++) {
var cleanedArray = $.grep(arrayOfArrays[i], function(element) {
return element != ""
});
if (cleanedArray.length == 0) {
arrayOfArrays.splice(i, 1);
}
}
return arrayOfArrays;
};

You can use $.grep :
multiArray = $.grep(multiArray, function(v){
return v.join('');
});
Fiddle : http://jsfiddle.net/scZcB/

on the fly:
var multiArray = [["","","",""],[1,2,3],["hello","dog","cat"],["","","",""]];
var outputArr = removeQuoteArrays(multiArray);
console.log(outputArr);
function removeQuoteArrays(arr) {
var outputArr = [];
for (var i = 0; i < arr.length; i++) {
var currArr = arr[i];
var isAllQuotes = true;
for (var j = 0; j < currArr.length; j++) {
if (currArr[j] != "") {
isAllQuotes = false;
break;
}
}
if (!isAllQuotes) {
outputArr.push(currArr);
}
}
return outputArr;
}
Here's a JSFiddle.

Create a new array instead.
// Only add if...
cleanedArray = multiArray.filter(function(arr){
// Some elements are not blank
return arr.some(function(e){ return e !== "" })
})

I added a length check to your function to break out of the loop if the index reaches the array length:
if (i >= arrayOfArrays.length)
break;
Which makes:
function cleanWhitespace(arrayOfArrays) {
var i;
var arrayOfArraysLength = arrayOfArrays.length;
for (i = 0; i < arrayOfArraysLength; i++) {
var cleanedArray = $.grep(arrayOfArrays[i], function(element) {
return element != ""
});
if (cleanedArray.length == 0) {
arrayOfArrays.splice(i, 1);
if (i >= arrayOfArrays.length)
break;
}
}
return arrayOfArrays;
};

var multiArray = [["","","",""],[1,2,3],["hello","dog","cat"],["","","",""]];
function cleanWhitespace(arrayOfArrays) {
for (var i = 0; i < arrayOfArrays.length; i++) {
var emptyElements = 0;
for (var j = 0; j < arrayOfArrays[i].length; j++ ) {
if (arrayOfArrays[i][j] === "") {
emptyElements++;
}
}
if (emptyElements === arrayOfArrays[i].length) {
arrayOfArrays.splice(i, 1);
}
}
return arrayOfArrays;
}
console.log(cleanWhitespace(multiArray));
http://jsfiddle.net/4Jfr9/

Arrays Splice Pop Shift Read

I create an array like so
var membersList = $('#chatbox_members' , avacweb_chat.doc.body).find('li');
var onlineUsers = [];
var offLineUsers = [];
for(var i =0;i<membersList.length;i++){
var name = $(membersList[i]).text().replace("#","");
onlineUsers.push(name);
}
alert(onlineUsers);
listedUsers would come out something like so [Mr.EasyBB,Tonight,Tomorrow,Gone];
Question is if I use a two for loops one outside a setInterval and one inside to compare-
var membersList = $('#chatbox_members' , _chat.doc.body).find('li');
var onlineUsers = [];
var offLineUsers= [];
for(var i =0;i<membersList.length;i++){
var name = $(membersList[i]).text().replace("#","");
onlineUsers.push(name);
}
var int = setInterval(function() {
var newMember = ('#chatbox_members' , _chat.doc.body).find('li');
for(var i =0;i<newMember.length;i++){
var name = $(newMember[i]).text().replace("#","");
offLineUsers.push(name);
}
Which then would get:
onlineUsers = [Mr.EasyBB,Tonight,Tomorrow,Gone];
offLineUsers = [Mr.EasyBB,Tonight];
So to get the offline users I want to basically replace onlineUsers with offLineUsers which then should return Tomorrow,Gone . Though I know that an object doesn't have the function to replace so how would I go about this?
I don't think the splice function would work since you need to have parameters, and pop or shift are beginning and end of array.

for(var i = 0 ; i < offLineUsers.length ; i++)
{
for(var j = 0 ; j < onlineUsers.length ; j++)
{
if(onlineUsers[j] == offLineUsers[i])
{
onlineUsers.splice(j,1);
}
}
}
Try this snippet.

If I have understand well, maybe that helps:
function bus_dup() {
for(var i = 0; i < offLineUsers.length; i++) {
onLineUsers.splice(onLineUsers.indexOf(offLineUsers[i]),1);
}
offLineUsers = [];
}

This should do what you are looking for on a modern browser, using array.filter
var onlineUsers = ["Mr.EasyBB", "Tonight", "Tomorrow", "Gone"];
var offLineUsers = ["Mr.EasyBB", "Tonight"];
function discord(online, offline) {
return online.filter(function (element) {
return offline.indexOf(element) === -1;
});
}
console.log(discord(onlineUsers, offLineUsers));
Output
["Tomorrow", "Gone"]
On jsfiddle
If you want the difference regardless of the order of attributes passed to the function then you could do this.
var onlineUsers = ["Mr.EasyBB", "Tonight", "Tomorrow", "Gone"];
var offLineUsers = ["Mr.EasyBB", "Tonight"];
function difference(array1, array2) {
var a = array1.filter(function (element) {
return array2.indexOf(element) === -1;
});
var b = array2.filter(function (element) {
return array1.indexOf(element) === -1;
});
return a.concat(b);
}
console.log(difference(onlineUsers, offLineUsers));
console.log(difference(offLineUsers, onlineUsers));
Output
["Tomorrow", "Gone"] 
["Tomorrow", "Gone"] 
On jsfiddle