We Keep Coding

What is the correct way to send large CSV Files to Server using jquery

I have a CSV file with ~20,000 records. I send each line using the $.post method to my server using the FileReader API.
The problem is that the browser is buffering each record before starting to send the data and this way is very slow. I want to send each line separately to show a progressbar where it counts the request number of each line.
As this solution is very slow I'm thinking there are must be other ways of doing this to make it faster. Many thanks to your ideas.
$("#form_file").change(function(e) {
if (e.target.files != undefined) {
var reader = new FileReader();
reader.onload = function(e) {
var rows = e.target.result.split("\n");
var index = rows[0];
index = index.split(";");
gesamt = rows.length - 1;
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
cells = row.split(";");
var dataset = {};
for (var ii = 0; ii < cells.length; ii++) {
var value = cells[ii];
var key = index[ii]
var printError = function(error, explicit) {
console.log(`[${explicit ? 'EXPLICIT' : 'INEXPLICIT'}] ${error.name}: ${error.message}`);
}
try {
dataset[key] = value;
} catch (e) {
if (e instanceof RangeError) {
if (e.message.toLowerCase().indexOf('invalid array') !== -1) {
printError(e, true);
} else {
printError(e, false);
}
} else {
printError(e, false);
}
}
}
console.log(dataset);
row = insertrow(dataset, i);
$('#progressbar').show();
$('#progressvalue').text(i + '/' + gesamt);
$('#progresstitle').text('(' + dataset.title + ')');
}
};
var test = reader.readAsText(e.target.files.item(0));
}
});
function insertrow(mydata, step) {
var token = "{{app.request.query.get('_token')}}";
mydata = JSON.stringify(mydata);
$.post('preferences/upload?_token=' + token, {
data: mydata
}, function(data) {
$('#info').show();
var html = data.message + '<br />';
$('#info').append(html);
}, "json");
}

javascript error in nifi executeScript

I am having an issue getting the javascript script for the executeScript nifi process to work and would appreciate help with this. The goal is to pass a flowfile which will contain a json object. I need to parse this json without knowing the content/fields prior and pass this along to write it out to the flowfile that is being passed out to the next process that is MergeContent and counts the number flowfiles.
Tried testing the script and got the following error:
nifi.script.ExecuteScript - ExecuteScript[id=bd6842e9-e3a4-4d88-a59d-
7da1d74d109b] ExecuteScript[id=bd6842e9-e3a4-4d88-a59d-7da1d74d109b]
failed to process due to
org.apache.nifi.processor.exception.ProcessException:
javax.script.ScriptException: <eval>:21:17 Expected : but found value
let value = json[key];
^ in <eval> at line number 21 at column number 17; rolling
back session: org.apache.nifi.processor.exception.ProcessException:
javax.script.ScriptException: <eval>:21:17 Expected : but found value
I am not very familiar with javascript so would appreciate the help.
flowFile = session.get();
if (flowFile != null) {
var StreamCallback =
Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "ops_testQueue";
flowFile = session.write(flowFile, new StreamCallback(function
(inputStream, outputStream) {
var content = IOUtils.toString(inputStream,
StandardCharsets.UTF_8); // message or content
var message_content = {};
try {
message_content = JSON.parse(content);
if(Array.isArray(message_content)){
}
Object.keys(message_content).forEach((key) => {
var value = json[key];
result.push(key + '=' + value);
var jkey = "," + "\"" + key + "\"" + '=' + value
});
line = line + jkey +
" value=" + "1"
+ " " + Date.now() * 1000000;
// Write output content
if (transformed_message) {
outputStream.write(line.getBytes(StandardCharsets.UTF_8));
}
} catch (e) {
error = true;
outputStream.write(content.getBytes(StandardCharsets.UTF_8));
}
}));
if (transformed_message.post_state) {
flowFile = session.putAttribute(flowFile, "type",
transformed_message.type);
}
if (error) {
session.transfer(flowFile, REL_FAILURE)
} else {
session.transfer(flowFile, REL_SUCCESS)
}
}
EDIT:
input to executeScript:
{"pID":"1029409411108724738",
"contentType":"text",
"published":"2018-08-14 16:48:23Z",
"crawled":"2018-08-14 12:48:33-04:00",
"ID":"765"}
output from executeScript:
ops_testQueue,"ID"=765 value=1 1534265314969999870
Am I missing something?

I saw a couple of things here:
I don't know if Nashorn (Java's JS Engine) supports the full lambda
syntax, I was able to get it to work by making the lambda a function
(see script below).
You refer to a json variable to get the value from a key, but I think you want message_content.
result is not defined, so you get an error when you push to it.
Here's an edited version of your script that I got to work the way I think you want it (but please correct me if I'm wrong):
flowFile = session.get();
if (flowFile != null) {
var StreamCallback =
Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "ops_testQueue";
flowFile = session.write(flowFile, new StreamCallback(function
(inputStream, outputStream) {
var content = IOUtils.toString(inputStream,
StandardCharsets.UTF_8); // message or content
var message_content = {};
try {
message_content = JSON.parse(content);
if(Array.isArray(message_content)){
}
var jkey = "";
Object.keys(message_content).forEach(function(key) {
var value = message_content[key];
//result.push(key + '=' + value);
jkey = "," + "\"" + key + "\"" + '=' + value
});
line = line + jkey +
" value=" + "1"
+ " " + Date.now() * 1000000;
// Write output content
if (transformed_message) {
outputStream.write(line.getBytes(StandardCharsets.UTF_8));
}
} catch (e) {
error = true;
log.error(e);
outputStream.write(content.getBytes(StandardCharsets.UTF_8));
}
}));
if (transformed_message.post_state) {
flowFile = session.putAttribute(flowFile, "type",
transformed_message.type);
}
if (error) {
session.transfer(flowFile, REL_FAILURE)
} else {
session.transfer(flowFile, REL_SUCCESS)
}
}

data = JSON.parse(data); syntax error unexpected end of input

I have a problem with my page. It works on localhost (WebServ). On my vps (Debian 8) it not work.
You can look: http://www.csgobeton.pl/withdraw.
When you do recaptcha it says "syntax error unexpected end of input".
In the console I can see there is a problem at line "data = JSON.parse(data);".
Please help me.
function loadLeft(opts) {
inlineAlert("", "Loading your inventory - please wait...");
var DIV = "<div class='placeholder matched' data-name='{0}' data-pos='{1}'
data - price = '{2}'
data - bot = '{3}' > ";
DIV += "<div class='slot {13}' data-view='{15}' data-name='{4}' data-
pos = '{5}'
data - price = '{6}'
data - bot = '{7}'
data - id = '{8}'
style = 'background-
image: url(\"{9}\")'>"; DIV += "<div class='name'>{10}</div>"; DIV +=
"<div class='price {11}'>{12}</div>"; DIV +=
"<div class='bot'>{14}</div>"; DIV += "</div></div>";
var IMG = "{0}/{1}fx{2}f";
var url = "";
if (DEPOSIT) {
url = "/get_inv?" + opts;
} else {
var g = grecaptcha.getResponse();
url = "http://www.csgobeton.pl/get_bank_safe?g-recaptcha-response=" + g;
}
$.ajax({
url: url,
success: function (data) {
$("#inlineAlert").html("data");
try {
data = JSON.parse(data);
if (data.success) {
console.log(data);
$("#left .reals").empty();
$("#right .reals").empty();
$("#right .bricks").removeClass("hidden");
$("#avail").html(formatNum(data.balance));
var count = data.items.length;
var eleA = [];
for (var i = 0; i < count; i++) {
var item = data.items[i];
var url = IMG.format(item.img, 110, 50);
var price_class = "ball-1";
if (DEPOSIT) {
price_class = "ball-0";
}
var slot_class = "";
var price_content = item.price;
if (price_content == 0) {
price_content = item.reject;
slot_class = "reject";
} else {
price_content = formatNum(price_content);
}
bot = 0;
if (item.botid) {
bot = item.botid;
}
var botLabel = "";
if (!DEPOSIT) {
botLabel = "Bot " + bot;
}
var ele = DIV.format(item.name, i, item.price, bot,
item.name, i, item.price, bot, item.assetid, url, item.name,
price_class,
price_content, slot_class, botLabel, item.view);
eleA.push(ele);
}
$("#left_number").html(count);
document.getElementById("left").getElementsByClassName(
"reals")[0].innerHTML = eleA.join('');
addPadding("#left", 6);
if (data.fromcache) {
inlineAlert("success", "Loaded " + count + " items from
cache - < a href = \
"javascript:loadLeft('nocache')\">reload inventory</a>"
);
} else {
inlineAlert("success", "Loaded " + count + " items.");
}
} else {
inlineAlert("error", data.error);
if (data.count > 0) {
cmd();
}
}
if (data.tid) {
showPending(data);
}
} catch (err) {
inlineAlert("error", "Javascript error22: " + err);
console.log(err.stack);
}
},
error: function (err) {
inlineAlert("error", "AJAX error: " + err.statusText);
},
});
}
index.php is:
case 'get_bank_safe':
if(!$user) exit(json_encode(array('success'=>false, 'error'=>'You must
login to access the widthdraw.')));
$g = curl('https://www.google.com/recaptcha/api/siteverify?secret=6Le-
6R8TAAAAAAjgt_wfQMOUCMxCCAWs-iFlP9T-&response='.$_GET['g-recaptcha-response']);
$g = json_decode($g, true);
if($g['success'] == true) {
$array =array('balance'=>$user['balance'],'error'=>'none','items'=>array(),'success'=>true);
mysql_query('SET CHARACTER SET utf8');
$sql = $db->query('SELECT * FROM `items` WHERE `status` = 1');
$prices = file_get_contents('prices.txt');
$prices = json_decode($prices, true);
while ($row = $sql->fetch()) {
$array['items'][] =
array('botid'=>$row['botid'],'img'=>'http://steamcommunity-a.akamaihd.net/economy/image/'.$row['img'],'name'=>$row['market_hash_name'],'ass
etid'=>$row['id'],'price'=>$prices['response']['items']
[$row['market_hash_name']]['value']*10,'reject'=>'unknown items');
}
exit(json_encode($array));
}
break;
The result of get_safe_bank is:
{"balance":"0","error":"none","items": [{"botid":"1","img":"http:\/\/steamcommunity-a.akamaihd.net\/economy\/image\/-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXU5A1PIYQNqhpOSV-fRPasw8rsUFJ5KBFZv668FFAuhqSaKWtEu43mxtbbk6b1a77Twm4Iu8Yl3bCU9Imii1Xt80M5MmD7JZjVLFH-6VnQJQ","name":"Chroma 2 Case","assetid":"8","price":0,"reject":"unknown items"},{"botid":"1","img":"http:\/\/steamcommunity-a.akamaihd.net\/economy\/image\/-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpoo6m1FBRp3_bGcjhQ09ulq5WYh8jiPLfFl2xU18h0juDU-MKljgLjqRVuaj-gLIKUdQdtMgvS-VK_wrvpgZ7quM_Im3Qw6Cdz4CzZgVXp1o7eGVz_","name":"USP-S | Lead Conduit (Well-Worn)","assetid":"26","price":0,"reject":"unknown items"}],"success":true}{"balance":"0","error":"none","items":[{"botid":"1","img":"http:\/\/steamcommunity-a.akamaihd.net\/economy\/image\/-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXU5A1PIYQNqhpOSV-fRPasw8rsUFJ5KBFZv668FFAuhqSaKWtEu43mxtbbk6b1a77Twm4Iu8Yl3bCU9Imii1Xt80M5MmD7JZjVLFH-6VnQJQ","name":"Chroma 2 Case","assetid":"8","price":0,"reject":"unknown items"},{"botid":"1","img":"http:\/\/steamcommunity-a.akamaihd.net\/economy\/image\/-9a81dlWLwJ2UUGcVs_nsVtzdOEdtWwKGZZLQHTxDZ7I56KU0Zwwo4NUX4oFJZEHLbXH5ApeO4YmlhxYQknCRvCo04DEVlxkKgpoo6m1FBRp3_bGcjhQ09ulq5WYh8jiPLfFl2xU18h0juDU-MKljgLjqRVuaj-gLIKUdQdtMgvS-VK_wrvpgZ7quM_Im3Qw6Cdz4CzZgVXp1o7eGVz_","name":"USP-S | Lead Conduit (Well-Worn)","assetid":"26","price":0,"reject":"unknown items"}],"success":true}
When I paste it instead data = JSON.parse(data); data it works correctly.

Correct order in for loop using Parse

I want to create a array containing objects, and I'm using Parse to query all the data.
However, the for loop which loops over the results doesn't does that in the correct order but randomly loops over the data. If I log i each iteration, the logs show different results every time.
Here is my code:
for (var i = 0; i < results.length; i++)
{
Parse.Cloud.useMasterKey();
// retrieve params
var objectid = results[i];
var self = request.params.userid;
// start query
var Payment = Parse.Object.extend("Payments");
var query = new Parse.Query(Payment);
query.get(objectid, {
success: function (payment) {
// get all the correct variables
var from_user_id = payment.get("from_user_id");
var to_user_id = payment.get("to_user_id");
var amount = payment.get("amount");
var createdAt = payment.updatedAt;
var note = payment.get("note");
var img = payment.get("photo");
var location = payment.get("location");
var status = payment.get("status");
var fromquery = new Parse.Query(Parse.User);
fromquery.get(from_user_id, {
success: function(userObject) {
var fromusername = userObject.get("name");
var currency = userObject.get("currency");
var toquery = new Parse.Query(Parse.User);
toquery.get(to_user_id, {
success: function(touser)
{
var tousername = touser.get("name");
if(tousername !== null || tousername !== "")
{
sendArray(tousername);
}
},
error: function(touser, error)
{
var tousername = to_user_id;
if(tousername !== null || tousername !== "")
{
sendArray(tousername);
}
}
});
function sendArray(tousername) {
var array = new Array();
// create the time and date
var day = createdAt.getDate();
var year = createdAt.getFullYear();
var month = createdAt.getMonth();
var hour = createdAt.getHours();
var minutes = createdAt.getMinutes();
// create the timestamp
var time = "" + hour + ":" + minutes;
var date = "" + day + " " + month + " " + year;
var associativeArray = {};
if(self == from_user_id)
{
fromusername = "self";
}
if(self == to_user_id)
{
tousername = "self";
}
associativeArray["from"] = fromusername;
associativeArray["to"] = tousername;
associativeArray["amount"] = amount;
associativeArray["currency"] = currency;
associativeArray["date"] = date;
associativeArray["time"] = time;
associativeArray["status"] = status;
if(note == "" || note == null)
{
associativeArray["note"] = null;
}
else
{
associativeArray["note"] = note;
}
if(img == "" || img == null)
{
associativeArray["img"] = null;
}
else
{
associativeArray["img"] = img;
}
if(location == "" || location == null)
{
associativeArray["location"] = null;
}
else
{
associativeArray["location"] = location;
}
array[i] = associativeArray;
if((i + 1) == results.length)
{
response.success(array);
}
},
error: function(userObject, error)
{
response.error(106);
}
});
},
error: function(payment, error) {
response.error(125);
}
});
}
But the i var is always set to seven, so the associative arrays are appended at array[7] instead of the correct i (like 1,2,3,4,5)
The reason that this is so important is because I want to order the payment chronologically (which I have done in the query providing the results).
What can I do to solve this issue?

Success is a callback that happens at a later point in time. So what happens is, the for loop runs 7 times and calls parse 7 times. Then after it has run each of parse success calls will be executed, they look at i which is now at 7.
A simple way to fix this is to wrap the whole thing in an immediate function and create a new closure for i. Something like this
for(var i = 0; i < results.length; i++){
function(iClosure) {
//rest of code goes here, replace i's with iClosure
}(i);
}
Now what will happen is that each success function will have access to it's own iClosure variable and they will be set to the value of i at the point they were created in the loop.

JS: Can not convert to object, childNodes related

Ok, feeling stupid here, but wondering what the problem is here exactly.
Although the function works as it should, I get this JS Error in Opera. Not sure about other browsers...
Uncaught exception: TypeError: Cannot
convert
'document.getElementById("shoutbox_area"
+ moduleId)' to object
oElement = document.getElementById("shoutbox_area"
+ moduleId).childNodes;
Here is the relevant code:
function appendShout(XMLDoc)
{
var shoutData = XMLDoc.getElementsByTagName("item");
var oElement = [];
if (shoutData.length > 0)
{
var moduleId = shoutData[0].getAttribute("moduleid");
if (shoutData[shoutData.length - 1].getAttribute("lastshout") != "undefined")
{
for (var i = 0; i < shoutData.length; i++)
if (shoutData[i].firstChild.nodeValue != 0)
document.getElementById("shoutbox_area" + moduleId).innerHTML += shoutData[i].firstChild.nodeValue;
oElement = document.getElementById("shoutbox_area" + moduleId).childNodes;
var i = oElement.length;
while (i--)
{
if (i % 2 == 0)
oElement[i].className = "windowbg2";
else
oElement[i].className = "windowbg";
}
oElement[oElement.length - 2].style.borderBottom = "1px black dashed";
}
}
}
Can someone please help me to understand why it is giving me an error here:
oElement = document.getElementById("shoutbox_area" + moduleId).childNodes;
Can I not assign an array to the childNodes?
EDIT:
This JS Error occurs when I try and delete a shout. The JS function for deleting a shout is this:
function removeShout(shout, moduleID)
{
var shoutContainer = shout.parentNode.parentNode;
var send_data = "id_shout=" + shout.id;
var url = smf_prepareScriptUrl(smf_scripturl) + "action=dream;sa=shoutbox;xml;" + "delete_shout;" + "canmod=" + canMod[moduleID] + ";" + sessVar + "=" + sessId;
sendXMLDocument(url, send_data);
var shoutID = 0;
while (shoutID !== null)
{
var shoutID = document.getElementById(shout.parentNode.id);
var moduleID = shoutID.parentNode.getAttribute("moduleid");
if (shoutID.parentNode.lastChild)
{
var url = smf_prepareScriptUrl(smf_scripturl) + "action=dream;sa=shoutbox;xml;get_shouts=" + (shoutID.parentNode.lastChild.id.replace("shout_", "") - 1) + ";membercolor=" + memberColor[moduleID] + ";maxcount=" + maxCount[moduleID] + ";shoutboxid=" + shoutboxID[moduleID] + ";textsize=" + textSize[moduleID] + ";parsebbc=" + parseBBC[moduleID] + ";moduleid=" + moduleID + ";maxcount=" + maxCount[moduleID] + ";canmod=" + canMod[moduleID] + ";" + sessVar + "=" + sessId;
getXMLDocument(url, appendShout);
}
element = shoutID.parentNode.childNodes;
var i = element.length;
while (i--)
{
if (i % 2 == 0)
element[i].className = "windowbg2";
else
element[i].className = "windowbg";
}
shoutID.parentNode.removeChild(shoutID);
}
}
Am using the following functions for the sending and getting the XMLHttpRequest as you may have noticed already in the removeShout function above:
// Load an XML document using XMLHttpRequest.
function getXMLDocument(sUrl, funcCallback)
{
if (!window.XMLHttpRequest)
return null;
var oMyDoc = new XMLHttpRequest();
var bAsync = typeof(funcCallback) != 'undefined';
var oCaller = this;
if (bAsync)
{
oMyDoc.onreadystatechange = function () {
if (oMyDoc.readyState != 4)
return;
if (oMyDoc.responseXML != null && oMyDoc.status == 200)
{
if (funcCallback.call)
{
funcCallback.call(oCaller, oMyDoc.responseXML);
}
// A primitive substitute for the call method to support IE 5.0.
else
{
oCaller.tmpMethod = funcCallback;
oCaller.tmpMethod(oMyDoc.responseXML);
delete oCaller.tmpMethod;
}
}
};
}
oMyDoc.open('GET', sUrl, bAsync);
oMyDoc.send(null);
return oMyDoc;
}
// Send a post form to the server using XMLHttpRequest.
function sendXMLDocument(sUrl, sContent, funcCallback)
{
if (!window.XMLHttpRequest)
return false;
var oSendDoc = new window.XMLHttpRequest();
var oCaller = this;
if (typeof(funcCallback) != 'undefined')
{
oSendDoc.onreadystatechange = function () {
if (oSendDoc.readyState != 4)
return;
if (oSendDoc.responseXML != null && oSendDoc.status == 200)
funcCallback.call(oCaller, oSendDoc.responseXML);
else
funcCallback.call(oCaller, false);
};
}
oSendDoc.open('POST', sUrl, true);
if ('setRequestHeader' in oSendDoc)
oSendDoc.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
oSendDoc.send(sContent);
return true;
}
Hopefully this is good enough, you can do a view source on it to see the actual HTML, but there are attributes that get added to the Shoutbox tags at runtime so as to be XHTML compliant, etc..
Please let me know if there is anything else you need?
Thanks :)

The code is breaking because shoutID is null in the second of these two lines, the second time through the loop:
var shoutID = document.getElementById(shout.parentNode.id);
var moduleID = shoutID.parentNode.getAttribute("moduleid");
The first of those lines is strange. Why not just use var shoutID = shout.parentNode;?
Also, the moduleId attribute seems to be nowhere around.
What are you trying to achieve with the while loop?