We Keep Coding

JSON.stringify not working with nested array of objects

I know this question has been asked but none of the solutions are working for me and I can't figure out what's wrong. I have an object with a nested array of objects I am stringifying but I get a blank array of the nested array when I use JSON.stringify on it.
This is a simplified version of the way I'm constructing the object. The main difference is that there is a for loop iterating through all the rows, here I am just manually creating 2 rows
// JAVASCRIPT
let obj = {};
obj['type'] = 'Setting';
obj['id'] = 1;
obj['import'] = parseCSV();
function parseCSV() {
let jsonData = [];
let row1 = {};
let row2 = {};
row1['date'] = '2022-01-01';
row1['amount'] = '30';
row2['date'] = '2022-01-02';
row2['amount'] = '50';
jsonData.push(row1);
jsonData.push(row2);
return jsonData;
}
console.log('RAW DATA', obj);
console.log('STRINGIFIED', JSON.stringify(obj));
The above outputs the correct stringified JSON
But the full version of my code gives me a blank array for import.
Both objects look identical to me. The culprit is somewhere in my parseCSV function, because when I use the simplified version above I get the correct stringified data, but I can't pinpoint where I'm wrong. Below is my full function.
function parseCSV(file) {
let filename = file.name;
let extension = filename.substring(filename.lastIndexOf('.')).toUpperCase();
if(extension == '.CSV') {
try {
let reader = new FileReader();
let jsonData = [];
let headers = [];
reader.readAsBinaryString(file);
reader.onload = function(e) {
let rows = e.target.result.split('\n');
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
}
return jsonData;
} catch(err) {
console.error('!! ERROR READING CSV FILE', err);
}
} else alert('PLEASE UPLOAD A VALID CSV FILE');*/
}
Thanks for the help!
EDIT
When I add await before parseCSV as #BJRINT's answer suggests I get a syntax error await is only valid in async function
async function submitForm(event) {
event.preventDefault();
let newItem = await gatherFormData(event.target);
return fetch('server.php', {
method: "POST",
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(newItem)
})
.then(checkError)
.then(data => parseData(data))
.catch(err => console.error('>> ERROR READING JSON DATA', err));
}
function gatherFormData(target) {
const inputs = target.querySelectorAll('input');
let obj = {};
inputs.forEach(function(input) {
if(intKeys.indexOf(input.name) >= 0) obj[input.name] = parseInt(input.value);
else if(curKeys.indexOf(input.name) >= 0) obj[input.name] = parseInt(parseFloat(input.value) * 100);
else if(chkKeys.indexOf(input.name) >= 0) input.checked ? obj[input.name] = 1 : obj[input.name] = 0;
else if(fileKeys.indexOf(input.name) >= 0 && input.files.length > 0) obj[input.name] = parseCSV(input.files[0]);
else obj[input.name] = input.value;
});
return obj;
}

The problem does not come from the stringify function. Since you are filling your array asynchronously (when the reader callback is executed) but returning your data first, it is empty.
You could wrap your function with a Promise that resolves when the reader callback function is finally executed, like so:
function parseCSV(file) {
return new Promise((resolve, reject) => {
let filename = file.name;
let extension = filename.substring(filename.lastIndexOf('.')).toUpperCase();
if(extension !== '.CSV')
return reject('PLEASE UPLOAD A VALID CSV FILE')
try {
let reader = new FileReader();
let jsonData = [];
let headers = [];
reader.readAsBinaryString(file);
reader.onload = function(e) {
let rows = e.target.result.split('\n');
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
return resolve(jsonData);
}
} catch(err) {
return reject('!! ERROR READING CSV FILE', err);
}
})
}
// calling the function
const data = await parseCSV(file)

The solution that worked for my specific case was to use a combination of BJRINT's answer and a timer to keep checking if the data had finished loading which I found here.
async function parseCSV(file) {
return await new Promise((resolve, reject) => {
let extension = file.name.substring(file.name.lastIndexOf('.')).toUpperCase();
if(extension !== '.CSV') reject('PLEASE UPLOAD A VALID CSV FILE');
try {
let reader = new FileReader();
reader.readAsText(file);
reader.onload = function(e) {
let jsonData = [];
let headers = [];
let rows = e.target.result.split(/\r\n|\r|\n/);
for(let i = 0; i < rows.length; i++) {
let cells = rows[i].split(',');
let rowData = {};
for(let j = 0; j < cells.length; j++) {
if(i == 0) headers.push(cells[j].trim());
else {
if(headers[j]) rowData[headers[j]] = cells[j].trim();
}
}
if(i != 0 && rowData['date'] != '') jsonData.push(rowData);
}
resolve(jsonData);
}
} catch(err) {
reject(err);
}
});
}
function submitForm(event) {
event.preventDefault();
showForm(false);
loading.classList.remove('hidden');
let ready = true;
const inputs = event.target.querySelectorAll('input');
let newItem = {};
let check = function() {
if(ready === true) {
console.log(newItem);
console.log(JSON.stringify(newItem));
return fetch('server.php', {
method: "POST",
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
body: JSON.stringify(newItem)
})
.then(checkError)
.then(data => parseData(data))
.catch(err => console.error('>> ERROR READING JSON DATA', err));
}
setTimeout(check, 1000);
}
inputs.forEach(function(input) {
if(intKeys.indexOf(input.name) >= 0) newItem[input.name] = parseInt(input.value);
else if(curKeys.indexOf(input.name) >= 0) newItem[input.name] = parseInt(parseFloat(input.value) * 100);
else if(chkKeys.indexOf(input.name) >= 0) input.checked ? newItem[input.name] = 1 : newItem[input.name] = 0;
else if(fileKeys.indexOf(input.name) >= 0 && input.files.length > 0) {
ready = false;
parseCSV(input.files[0]).then(data => {
ready = true;
newItem[input.name] = data;
});
}
else newItem[input.name] = input.value;
});
check();
}

Threads module giving error of channel closed

I am using the npm xlsx (lib/parserScripts/readExcel.js)
and threads module to read a large excel file.
This works fine for the first time but if I simultaneously upload another large file then I get an error
Error: channel closed
at ChildProcess.target.send (internal/child_process.js:554:16)
at Worker.send (/app/node_modules/threads/lib/worker.node/worker.js:108:16)...
This is maybe due to the previous threads are still processing /have not been killed hence when a new pool is made for another request the previous threads are still busy processing.
How to solve this? Do I have to manually terminate the threads in the below piece of code? If so then how?
index.js
parseFile: ['fileHeaders', (results, cb) => {
const excelParserScript = __dirname + '/../lib/parserScripts/readExcel';
const worksheetIndex = 3;
const params = {
file.path,
worksheetIndex
}
// using worker process
// result will be of the type {error: false, message: '', data: {}}
lib.miniWorker.bufferedJob(excelParserScript, params, (err, result) => {
lib/miniworker.js
const Threads = require('threads');
const Pool = Threads.Pool;
const workerPool = new Pool();
module.exports = class JobManager {
static bufferedJob(pathToScript, params, callback){
workerPool
.run(pathToScript)
.send(params)
.on('done', (result, input) => {
console.log(`Worker Job done: ${pathToScript} `);
callback(null, result);
})
.on('error', (job, error) => {
console.log(`Error in executing Worker Job: ${pathToScript}`);
callback(job || error);
})
}
}
lib/parserScripts/readExcel.js
module.exports = function(input, done) {
const XLSX = require('xlsx');
let workbook;
const path = input.path;
const worksheetIndex = input.worksheetIndex;
const expectedHeaders = input.expectedHeaders || [];
const options = {};
if (expectedHeaders.length > 0) {
options.header = expectedHeaders;
}
const response = {
error: false,
message: '',
data: {}
}
try {
workbook = XLSX.readFile(path, {});
const sheet = workbook['Sheets'][workbook.SheetNames[worksheetIndex]];
const headers = getHeaders(sheet);
const fileData = XLSX.utils.sheet_to_json(workbook['Sheets'][workbook.SheetNames[worksheetIndex]], options);
response.data = fileData;
response.headers = headers;
return done(response)
} catch (err) {
response.error = true;
response.messsage = 'Error in reading the file';
return done(response);
}
function getHeaders(sheet) {
var header = 0, offset = 1;
var hdr = [];
var o = {};
if (sheet == null || sheet["!ref"] == null) return [];
var range = o.range !== undefined ? o.range : sheet["!ref"];
var r;
if (o.header === 1) header = 1;
else if (o.header === "A") header = 2;
else if (Array.isArray(o.header)) header = 3;
switch (typeof range) {
case 'string':
r = safe_decode_range(range);
break;
case 'number':
r = safe_decode_range(sheet["!ref"]);
r.s.r = range;
break;
default:
r = range;
}
if (header > 0) offset = 0;
var rr = XLSX.utils.encode_row(r.s.r);
var cols = new Array(r.e.c - r.s.c + 1);
for (var C = r.s.c; C <= r.e.c; ++C) {
cols[C] = XLSX.utils.encode_col(C);
var val = sheet[cols[C] + rr];
switch (header) {
case 1:
hdr.push(C);
break;
case 2:
hdr.push(cols[C]);
break;
case 3:
hdr.push(o.header[C - r.s.c]);
break;
default:
if (val === undefined) continue;
hdr.push(XLSX.utils.format_cell(val));
}
}
return hdr;
}
function safe_decode_range(range) {
var o = {s: {c: 0, r: 0}, e: {c: 0, r: 0}};
var idx = 0, i = 0, cc = 0;
var len = range.length;
for (idx = 0; i < len; ++i) {
if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
idx = 26 * idx + cc;
}
o.s.c = --idx;
for (idx = 0; i < len; ++i) {
if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
idx = 10 * idx + cc;
}
o.s.r = --idx;
if (i === len || range.charCodeAt(++i) === 58) {
o.e.c = o.s.c;
o.e.r = o.s.r;
return o;
}
for (idx = 0; i != len; ++i) {
if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
idx = 26 * idx + cc;
}
o.e.c = --idx;
for (idx = 0; i != len; ++i) {
if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
idx = 10 * idx + cc;
}
o.e.r = --idx;
return o;
}
}

This works fine for the first time but if I simultaneously upload another large file then I get an error.
you should upload a different file name like a final01.xlsx and then rename it to final.xlsx
The reason being when you upload file the readfile cant finish as writing a file locks the file and change content.
If upload file means you are simultaneously reading another large file in node.js code ignore my comment.

The issue is because of the older version module of threads. Updating to the new version and using the updated API which is not event-based can solve the purpose.
https://github.com/andywer/threads.js/issues/164
However, if you want to correct the event-based code(from older version) this is what you need to do (kill the threads after the event gets completed).
const Threads = require('threads');
const Pool = Threads.Pool;
module.exports = class JobManager {
static bufferedJob(pathToScript, params, callback){
let workerPool = new Pool();
workerPool
.run(pathToScript)
.send(params)
.on('done', (result, input) => {
console.log(`Worker Job done: ${pathToScript} `);
callback(null, result);
workerPool.killAll();
workerPool = null ;
})
.on('error', (job, error) => {
console.log(`Error in executing Worker Job: ${pathToScript}`);
callback(job || error);
workerPool.killAll();
workerPool = null ;
}).on('abort', (job, error)=>{
console.log(`Abort Worker Job: ${pathToScript}, Error : ${error}`);
callback(job || error);
workerPool.killAll();
workerPool = null ;
}).on('finished', ()=>{
console.log('Everything done, shutting down the thread pool.');
workerPool.killAll();
});
}
}

Highlighting works initially but does not when hash changes

I am developing an extension, which is about fetching the list of topics from the server and find if those topics match with the currently opened Gmail messages or not, if found then highlight that topic otherwise don't.
But if already 6 topics are matched, then it should not check or highlight other topics. This one is working but now I have a problem like if I go back from the current message and again come to that message then highlight won't be shown. Also if I open another message, the highlight is not done.
If I remove the code of counter check from the following snippet it works but this will highlight all the topics that are matched instead of just max 6 topics.
var count = 1;
var highlightAllWords = function(topics) {
Object.keys(topics.topics).forEach(function(topic) {
if (count <= 6) {
highlightTopic(topic);
if (topic !== null || !topic.length) {
count += 1;
}
}
});
};
// init highlight CSS
var ruleExistenceDict = {};
var sheet = (function() {
var style = document.createElement('style');
style.appendChild(document.createTextNode('')); // WebKit hack ##
document.head.appendChild(style);
return style.sheet;
})();
var topicData = {
topics: {
hostname: 4,
cto: 19,
aws: 382,
its: 26,
repo: 15,
unsubscribe: 65,
bitbucket: 313,
having: 28,
devops: 414,
frontend: 25,
stepin: 105,
username: 121,
deployed: 24,
vimeo: 460,
gmail: 156,
rds: 486,
clicked: 9,
lai: 850
}
};
function fetchTopics() {
// api call will be done here but for now its done with dummy object
searchPage(topicData);
}
function searchPage(topics) {
highlightAllWords(topics);
}
var count = 1;
var highlightAllWords = function(topics) {
Object.keys(topics.topics).forEach(function(topic) {
if (count <= 6) {
highlightTopic(topic);
if (topic !== null || !topic.length) {
count += 1;
}
}
});
};
function highlightTopic(topic) {
// let found = 0;
let isCompleted = false;
if (topic == null || topic.length === 0) return;
var topicRegex = new RegExp(topic, 'gi');
var treeWalker = document.createTreeWalker(
document.body,
NodeFilter.SHOW_TEXT, {
acceptNode: function(node) {
var result = NodeFilter.FILTER_SKIP;
if (topicRegex.test(node.nodeValue)) {
// found += 1;
// if (found <= 6) {
result = NodeFilter.FILTER_ACCEPT;
return result;
// }
}
}
},
false
);
var skipTagName = {
NOSCRIPT: true,
SCRIPT: true,
STYLE: true
};
var nodeList = [];
// let count = 1;
console.log('count near nextNode', count);
while (treeWalker.nextNode()) {
if (!skipTagName[treeWalker.currentNode.parentNode.tagName]) {
nodeList.push(treeWalker.currentNode);
// count = count + 1;
// console.log('count:' + count);
}
}
nodeList.forEach(function(n) {
var rangeList = [];
// find sub-string ranges
var startingIndex = 0;
do {
// console.log(word, startingIndex, n.parentNode, n.textContent);
startingIndex = n.textContent.indexOf(topic, startingIndex + 1);
if (startingIndex !== -1) {
var topicRange = document.createRange();
topicRange.setStart(n, startingIndex);
topicRange.setEnd(n, startingIndex + topic.length);
rangeList.push(topicRange);
}
} while (startingIndex !== -1);
// highlight all ranges
rangeList.forEach(function(r) {
highlightRange(r);
});
});
}
var highlightRange = function(range) {
const bgColorCode = '#000000';
var anchor = document.createElement('A');
var selectorName = (anchor.className = 'highlighted_text');
anchor.classList.add('highlighted_text');
if (!ruleExistenceDict[bgColorCode]) {
sheet.insertRule(
[
'.',
selectorName,
' { background: #',
bgColorCode,
' !important; }'
].join(''),
0
);
ruleExistenceDict[bgColorCode] = true;
console.log(sheet);
}
anchor.appendChild(range.extractContents());
anchor.href = `https://app.com/profile/topics/${range.extractContents()}`;
range.insertNode(anchor);
};
Here is the full code:
https://gist.github.com/MilanRgm/5d6b9861be1326ba8b049ccfb6c3b376

You should declare the count variable inside the function, so that every time you refresh the page, the count will start from 1 again. Please update your code as follows:
var highlightAllWords = function(topics) {
var count = 1;
Object.keys(topics.topics).forEach(function(topic) {
if (count <= 6) {
highlightTopic(topic);
if (topic !== null || !topic.length) {
count += 1;
}
}
});
};

$.whenall with Ajax

I have a function for returning a feed which is retrieved by an AJAX-Call and now want to do something after a few of these asynchron requests have been done.
doit() is the function I call at first.
I am sorry for not providing a url, but it is an internal server.
Here is my code:
function grabFollowedCommunityPageFeed(page, cCallback) {
$.ajax({
url: "blabla.com&page=" + page,
method: "GET",
contentType: "application/atom+xml"
}).always(function(xhr, ignore, thrownMessage) {
var totalResults = 0;
if ((200 === thrownMessage.status) && (xhr)) {
totalResults = parseInt($(xhr).find("totalResults").first().text()) || -1;
}
if (cCallback && $.isFunction(cCallback)) {
cCallback({feed: xhr, resultCount: totalResults});
}
});
}
function grabFollowedCommunitiesFeeds(pagecount) {
var i = 1,
deferredArr = [];
for (i = 1; i < pagecount; i += 1) {
grabFollowedCommunityPageFeed(i, function callback(resultObj) {
deferredArr[i] = new $.Deferred();
deferredArr[i].resolve(resultObj);
});
}
return deferredArr;
}
function doit() {
var allCommunityFeedObjects = [],
allCommunityFeedObjectsCount = 0,
deferredObj = [];
(function initialReadFollowedCommunityFeedPages() {
grabFollowedCommunityPageFeed(1, function(requestObj) {
allCommunityFeedObjectsCount = requestObj.resultCount;
var tEntries = $(requestObj.communityFeed).find("entry"),
el$;
$.each(tEntries, function(ignore, el) {
el$ = $(el);
if (!($.inArray(el$, allCommunityFeedObjects) !== -1)) {
allCommunityFeedObjects.push(el$);
}
});
deferredObj = grabFollowedCommunitiesFeeds(allCommunityFeedObjectsCount) || [];
$.whenAll.apply($, deferredObj).always(function(allCommunityFeeds) {
var k = allCommunityFeeds;
// union k with allCommunityFeedObjects
});
});
})();
}
This line seems to be fine as well and I have checked it:
deferredArr[i].resolve(resultObj);
The problem is that allCommunityFeeds parameter is undefined in
$.whenAll.apply($, deferredObj).always(function(allCommunityFeeds)
and that means there is something wrong. Can you help me?

How can Actionhero receive the parameter without inputs defined?

I'm using ActionHero in node.js and Angular.js.
I am trying images send to ActionHero using $http method.
but I don't know How many images are made.
so I can't define the parameter names on action in ActionHero.
below is my source.
First. images are in object, so I change object to each parameter.
insert: function (param, next) {
var url = settings.apiUrl + "/api/online/productAdd";
var vdata = {
img_objects :param.img_objects
};
angular.forEach(param.img_objects, function (v, k) {
vdata['img_file'+(k)] = v.files;
});
commonSVC.sendUrlFile("POST", url, vdata, function (state, data) {
next(state, data);
});
}
Second. make formData in sendUrlFile like source below. and then send to actionHero.
var promise = $http({
method: method,
url: url,
headers: {
'Content-Type': undefined
},
data: params,
transformRequest: function (data) {
var formData = new FormData();
angular.forEach(data, function (value, key) {
if(angular.isObject(value)){
if(value.lastModified > 0 && value.size > 0){
formData.append(key, value);
}else{
formData.append(key, JSON.stringify(value));
}
}else{
formData.append(key, value);
}
});
return formData;
}
});
Third. ActionHero is received. but parameter isn't defined so ActionHero can't receive.
exports.productAdd = {
name: 'online/productAdd',
inputs: {
I don't know How Many Images are made? 1~10? or 1~100?
},
authenticate: true,
outputExample: {
'result':'success'
}
So I have two Questions:
How can actionhero receive the parameter without inputs defined?
Can I object with Image Data send to ActionHero by Ajax?
Thank You.

I change reduceParams function in actionProcessor.js.
api.actionProcessor.prototype.reduceParams = function(){
var self = this;
var inputNames = [];
if(self.actionTemplate.inputs){
inputNames = Object.keys(self.actionTemplate.inputs);
}
// inputs * 확인 2017-01-20 Eddy
var multi = [];
var strArray;
for(var v in inputNames){
if(inputNames[v].indexOf("*") != -1){
strArray = inputNames[v].split('*');
multi.push(strArray[0]);
}
}
var multiLength = multi.length;
var flag;
if(api.config.general.disableParamScrubbing !== true){
for(var p in self.params){
flag = true;
if(multiLength > 0){
for(var i=0; i<multiLength; i++){
if(p.indexOf(multi[i]) != -1){
flag = false;
}
}
}
if(flag){
if(api.params.globalSafeParams.indexOf(p) < 0 && inputNames.indexOf(p) < 0){
delete self.params[p];
}
}
}
}
};
i can define on inputs like below.
'img_*' : {required: false}

and Then I make middleware
var actionHeroMiddleware = {
name: '-',
global: true,
priority: 1000,
preProcessor: function(data, next) {
api.actionProcessor.prototype.reduceParams = function(){
var self = this;
var inputNames = [];
if(self.actionTemplate.inputs){
inputNames = Object.keys(self.actionTemplate.inputs);
}
// inputs * 확인 2017-01-20 Eddy
var multi = [];
var strArray;
for(var v in inputNames){
if(inputNames[v].indexOf("*") != -1){
strArray = inputNames[v].split('*');
multi.push(strArray[0]);
}
}
var multiLength = multi.length;
var flag;
if(api.config.general.disableParamScrubbing !== true){
for(var p in self.params){
flag = true;
if(multiLength > 0){
for(var i=0; i<multiLength; i++){
if(p.indexOf(multi[i]) != -1){
flag = false;
}
}
}
if(flag){
if(api.params.globalSafeParams.indexOf(p) < 0 && inputNames.indexOf(p) < 0){
delete self.params[p];
}
}
}
}
};
next();
},
stop: function(api, next) {
next();
}
};
api.actions.addMiddleware(actionHeroMiddleware);
next();