We Keep Coding

What is the correct way to send large CSV Files to Server using jquery

I have a CSV file with ~20,000 records. I send each line using the $.post method to my server using the FileReader API.
The problem is that the browser is buffering each record before starting to send the data and this way is very slow. I want to send each line separately to show a progressbar where it counts the request number of each line.
As this solution is very slow I'm thinking there are must be other ways of doing this to make it faster. Many thanks to your ideas.
$("#form_file").change(function(e) {
if (e.target.files != undefined) {
var reader = new FileReader();
reader.onload = function(e) {
var rows = e.target.result.split("\n");
var index = rows[0];
index = index.split(";");
gesamt = rows.length - 1;
for (var i = 1; i < rows.length; i++) {
var row = rows[i];
cells = row.split(";");
var dataset = {};
for (var ii = 0; ii < cells.length; ii++) {
var value = cells[ii];
var key = index[ii]
var printError = function(error, explicit) {
console.log(`[${explicit ? 'EXPLICIT' : 'INEXPLICIT'}] ${error.name}: ${error.message}`);
}
try {
dataset[key] = value;
} catch (e) {
if (e instanceof RangeError) {
if (e.message.toLowerCase().indexOf('invalid array') !== -1) {
printError(e, true);
} else {
printError(e, false);
}
} else {
printError(e, false);
}
}
}
console.log(dataset);
row = insertrow(dataset, i);
$('#progressbar').show();
$('#progressvalue').text(i + '/' + gesamt);
$('#progresstitle').text('(' + dataset.title + ')');
}
};
var test = reader.readAsText(e.target.files.item(0));
}
});
function insertrow(mydata, step) {
var token = "{{app.request.query.get('_token')}}";
mydata = JSON.stringify(mydata);
$.post('preferences/upload?_token=' + token, {
data: mydata
}, function(data) {
$('#info').show();
var html = data.message + '<br />';
$('#info').append(html);
}, "json");
}

Openlayers 3 with new ol.Vector returns [Infinity, Infinity, -Infinity, -Infinity]

I have a search form to call a Solr index, filled with geolocations:
jQuery('.form-submit', element).click(function (e) {
e.preventDefault();
search();
});
function search() {
useBBOX = false;
combinedExtent = ol.extent.createEmpty();
data.map.getLayers().forEach(function (layer, index, array) {
if (layer.getSource() instanceof ol.source.Vector) {
var source = layer.getSource().getSource();
var url = data.opt.urls[source.get('machineName')];
var newSource = new ol.source.Vector({
loader: getLoader(data, url),
format: new ol.format.GeoJSON(),
strategy: ol.loadingstrategy.bbox,
reloadOnZoomChange: true,
reloadOnExtentChange: true
});
newSource.set('machineName', source.get('machineName'));
var newCluster = new ol.source.Cluster({
source: newSource,
distance: 200
});
layer.setSource(newCluster);
}
});
}
function getLoader(data, url) {
return function (extent, resolution, projection) {
var bbox = ol.proj.transformExtent(extent, data.map.getView().getProjection(), 'EPSG:4326');
var params = {};
if (data.opt.paramForwarding) {
var get_params = location.search.substring(location.search.indexOf('?') + 1).split('&');
jQuery.each(get_params, function (i, val) {
if (val.length) {
var param = val.split('=');
params[decodeURIComponent(param[0])] = (param[1] !== undefined) ? decodeURIComponent(param[1].replace(/\+/g, ' ')) : '';
}
})
}
if (useBBOX == true) {
params.bbox = bbox.join(',');
params.zoom = data.map.getView().getZoom();
}
var searchQuery = jQuery('#input-search-address').val();
if (searchQuery != 'undefined' && searchQuery != null) {
url = url.substr(0, url.lastIndexOf("/") + 1);
url = url + searchQuery;
}
jQuery(document).trigger('openlayers.bbox_pre_loading', [{
'url': url,
'params': params,
'data': data
}]);
var that = this;
jQuery.ajax({
url: url,
data: params,
success: function (responsdata) {
var features = that.getFeaturesInExtent(extent);
jQuery(features).each(function (i, f) {
that.removeFeature(f);
});
var format = new ol.format.GeoJSON();
var features = format.readFeatures(responsdata, {featureProjection: projection});
that.addFeatures(features);
that._loadingFeatures = false;
if (!ol.extent.isEmpty(that.getExtent())) {
combinedExtent = ol.extent.extend(combinedExtent, that.getExtent());
if (useBBOX == false) {
useBBOX = true;
}
}
}
});
};
}
Basically, it fetches 3 layers, each containing a number of markers. I'ld like to autozoom the map based on those markers. Therefore I'm looking for the extent. The combinedExtent does contain all correct extents...
... but when I add data.map.getView().fit(combinedExtent, data.map.getSize()) INSIDE the getLoader function, it's not working. I looks like only 1 extent get plotted on the map.
Whenever I try to log the combinedExtent in the search() function, I get a weird error...
Google told me I had to wait until the getState() of newSource was ready, but that didn't work out...
So, I'm looking for a solution. My guess would be the use of the ajax return in getLoader...

I just had this issue, so I have a function to listen until the source is ready and finished loading before giving a count of features (otherwise it would update the log with every iteration). Just change my source name (any variable with "parcelquery in it) with yours and hopefully it will at least put you in the right direction.
var listenerKey = wfsSource_parcelquery.on('change', function(e) {
if (wfsSource_parcelquery.getState() == 'ready') { //says source is done loading
var featureCount = wfsSource_parcelquery.getFeatures().length; //getting number of features resulting from query
ol.Observable.unByKey(listenerKey);
// use vectorSource.unByKey(listenerKey) instead
// if you do use the "master" branch of ol3
}
alert("Your query returned "+ featureCount + " results.");
var extent = lyr_parcelquery.getSource().getExtent();
console.log(extent);
map.getView().fit(extent, map.getSize());
});

Clear Marker Layers (Leaflet )

I am using Leaflet library and stuck with following issue:
To generate Map i am calling map function on button click.
So on each generatemap function call, I want to clear pregenerated markers.
function generatefilterrecord(orgid,defservice,defavail,defowner,defhealthfacility) {
$("#content1").hide();
$("#map1").show();
$("#mapA1").hide();
$("#footer").hide();
jQuery('#sel').html('');
$(".w3-card-4").remove();
var addressjoin=[],ouid=[],availspecialitiesjoin,availspecialities=[],avaialabilityjoin,name=[],address=[],pincode=[],village=[],mobile=[],special=[],notspecial=[],hfacilities=[],nothfacilities=[],schemes=[],notschemes=[],contactpname=[],contactpnumber=[];
var toAdd,spec=[],email=[],specialjoin,notspecialjoin,notspec=[],owner=[],notowner=[],hfacilitiesjoin,nothfacilitiesjoin,schemesjoin,notschemesjoin,hfschemes=[],nothfschemes=[];
var healthfac="",ownership=[],availspecialiti=[];
var arrayMap = [];latitude=[];longitude=[];
$.getJSON("../../api/analytics/events/query/tzR46QRZ6FJ.json?stage=o6ps51YxGNb&dimension=pe:THIS_YEAR&dimension=ou:"+orgid+"&dimension=l8VDWUvIHmv&dimension=KOhqEw0UKxA&dimension=xjJR4dTmn4p&dimension=wcmHow1kcBi&dimension=pqVIj8NyTXb&dimension=g7vyRbNim1K&dimension=Gx4VSNet1dC&dimension=bUg8a8bAvJs&dimension="+defservice+"&dimension="+defavail+"&dimension="+defowner+"&dimension="+defhealthfacility+"&dimension=ZUbPsfW6y0C&dimension=CAOM6riDtfU&dimension=YL7OJoQCAmF&dimension=vJO1Jac84Ar&dimension=kF8ZJYe9SJZ&dimension=tNhLX6c7KHp&dimension=bVENUe0eDsO&displayProperty=NAME", function (data) {
console.log("../../api/analytics/events/query/tzR46QRZ6FJ.json?stage=o6ps51YxGNb&dimension=pe:THIS_YEAR&dimension=ou:"+orgid+"&dimension=l8VDWUvIHmv&dimension=KOhqEw0UKxA&dimension=xjJR4dTmn4p&dimension=wcmHow1kcBi&dimension=pqVIj8NyTXb&dimension=g7vyRbNim1K&dimension=Gx4VSNet1dC&dimension=bUg8a8bAvJs&dimension="+defservice+"&dimension="+defowner+"&dimension="+defhealthfacility+"&dimension=jXCd8k2841l&dimension=RkP5neDLbHv&dimension=avHST8wLPnX&dimension=txl9e6UJFP4&dimension=ZUbPsfW6y0C&dimension=CAOM6riDtfU&dimension=YL7OJoQCAmF&dimension=vJO1Jac84Ar&dimension=kF8ZJYe9SJZ&dimension=tNhLX6c7KHp&dimension=bVENUe0eDsO&displayProperty=NAME");
var constants={key:name, value: value}
analyticsMap = calculateIndex(data.headers,analyticsMap);
if(data.rows.length==0)
{
alert("No result found for above selection");
}
for(var k=0;k<data.rows.length;k++){
arrayMap["special"] = special;
arrayMap["name"] = name;
arrayMap["address"] = addressjoin;
arrayMap["pincode"] = pincode;
arrayMap["village"] = village;
arrayMap["mobile"] = mobile;
arrayMap["notspecial"] = notspecial;
arrayMap["hfacilities"] = hfacilities;
arrayMap["nothfacilities"] = nothfacilities;
arrayMap["schemes"] = schemes;
arrayMap["notschemes"] = notschemes;
arrayMap["contactpname"] = contactpname;
arrayMap["contactpnumber"] = contactpnumber;
arrayMap["availspecialities"] = availspecialities;
arrayMap["ownership"] = ownership;
arrayMap["ouid"] = ouid;
for (var j=0;j<analyticsMap.length;j++){
if (analyticsMap[j].index > 0){
var value = data.rows[k][analyticsMap[j].index];
if (value == "1"){
value = data.headers[analyticsMap[j].index].column;
}
if (!value || value == "0"){
value = "";
}
if(arrayMap[analyticsMap[j].arrayName]){
arrayMap[analyticsMap[j].arrayName].push(value);
}
}
}
specialjoin = myJoin(special);
availspecialitiesjoin = myJoin(availspecialities);
notspecialjoin = myJoin(notspecial);
hfacilitiesjoin = myJoin(hfacilities);
nothfacilitiesjoin = myJoin(nothfacilities);
schemesjoin = myJoin(schemes);
notschemesjoin = myJoin(notschemes);
spec.push(specialjoin);
notspec.push(notspecialjoin);
owner.push(hfacilitiesjoin);
availspecialiti.push(availspecialitiesjoin);
notowner.push(nothfacilitiesjoin);
hfschemes.push(schemesjoin);
nothfschemes.push(notschemesjoin);
availspecialities=[];
special = [];
notspecial = [];
hfacilities = [];
nothfacilities = [];
schemes = [];
notschemes = [];
}
var header = {
"Authorization": "Basic " + btoa( "homepage" + ':' + "Homepage123#123" )
};
for (var i = 0; i < name.length; i++) {
$.ajax({
async: false,
type: "GET",
dataType: "json",
contentType: "application/json",
header: header,
url: '../../api/organisationUnits/' + ouid[i] + '.json?fields=[id,name,coordinates]',
success: function (response) {
var coordinates = JSON.parse(response.coordinates);
latitude.push(coordinates[0]);
longitude.push(coordinates[1]);
},
error: function (response) {
}
});
}
for (var i = 0; i < name.length; i++) {
if(ownership[i]=="Public")
{
L.marker([longitude[i], latitude[i]], {icon: blueMarker}).addTo(map1).bindPopup(name[i]+","+"</br><strong>Contact:</strong> "+ mobile[i]+ ",</br> <strong>Schemes:</strong>"+hfschemes[i]+", </br><strong>Availabilities:</strong> "+availspecialiti[i]).openPopup();
}
else if(ownership[i]=="Private")
{
L.marker([longitude[i], latitude[i]], {icon: redMarker}).addTo(map1).bindPopup(name[i]).openPopup();
}
}
});
}
Here map never generates but if i remove marker.clearLayers(); then it is coming fine but problem is it appends every result so I want to clear the last generated markers.

Instead of adding the markers directly to your map, add them to a L.LayerGroup
Whenever you want, you can remove your markers calling the clearLayers method
var layerGroup = L.layerGroup().addTo(map);
// create markers
L.marker().addTo(layerGroup);
// remove all the markers in one go
layerGroup.clearLayers();

You can use this
$(".leaflet-marker-icon").remove();
$(".leaflet-popup").remove();

All simple:
map.eachLayer((layer) => {
layer.remove();
});
from https://leafletjs.com/reference-1.0.3.html#map-event

as the code provided by psyopus removes every layer (base maps, markers..etc); I needed just to remove the markers, the marker layer object has key of'_latlng'. Thus, I check if the object has it, so remove it:
map.eachLayer((layer) => {
if(layer['_latlng']!=undefined)
layer.remove();
});
Disclaimer: I dont know if there are other layer objects with this parameter key.

Correct order in for loop using Parse

I want to create a array containing objects, and I'm using Parse to query all the data.
However, the for loop which loops over the results doesn't does that in the correct order but randomly loops over the data. If I log i each iteration, the logs show different results every time.
Here is my code:
for (var i = 0; i < results.length; i++)
{
Parse.Cloud.useMasterKey();
// retrieve params
var objectid = results[i];
var self = request.params.userid;
// start query
var Payment = Parse.Object.extend("Payments");
var query = new Parse.Query(Payment);
query.get(objectid, {
success: function (payment) {
// get all the correct variables
var from_user_id = payment.get("from_user_id");
var to_user_id = payment.get("to_user_id");
var amount = payment.get("amount");
var createdAt = payment.updatedAt;
var note = payment.get("note");
var img = payment.get("photo");
var location = payment.get("location");
var status = payment.get("status");
var fromquery = new Parse.Query(Parse.User);
fromquery.get(from_user_id, {
success: function(userObject) {
var fromusername = userObject.get("name");
var currency = userObject.get("currency");
var toquery = new Parse.Query(Parse.User);
toquery.get(to_user_id, {
success: function(touser)
{
var tousername = touser.get("name");
if(tousername !== null || tousername !== "")
{
sendArray(tousername);
}
},
error: function(touser, error)
{
var tousername = to_user_id;
if(tousername !== null || tousername !== "")
{
sendArray(tousername);
}
}
});
function sendArray(tousername) {
var array = new Array();
// create the time and date
var day = createdAt.getDate();
var year = createdAt.getFullYear();
var month = createdAt.getMonth();
var hour = createdAt.getHours();
var minutes = createdAt.getMinutes();
// create the timestamp
var time = "" + hour + ":" + minutes;
var date = "" + day + " " + month + " " + year;
var associativeArray = {};
if(self == from_user_id)
{
fromusername = "self";
}
if(self == to_user_id)
{
tousername = "self";
}
associativeArray["from"] = fromusername;
associativeArray["to"] = tousername;
associativeArray["amount"] = amount;
associativeArray["currency"] = currency;
associativeArray["date"] = date;
associativeArray["time"] = time;
associativeArray["status"] = status;
if(note == "" || note == null)
{
associativeArray["note"] = null;
}
else
{
associativeArray["note"] = note;
}
if(img == "" || img == null)
{
associativeArray["img"] = null;
}
else
{
associativeArray["img"] = img;
}
if(location == "" || location == null)
{
associativeArray["location"] = null;
}
else
{
associativeArray["location"] = location;
}
array[i] = associativeArray;
if((i + 1) == results.length)
{
response.success(array);
}
},
error: function(userObject, error)
{
response.error(106);
}
});
},
error: function(payment, error) {
response.error(125);
}
});
}
But the i var is always set to seven, so the associative arrays are appended at array[7] instead of the correct i (like 1,2,3,4,5)
The reason that this is so important is because I want to order the payment chronologically (which I have done in the query providing the results).
What can I do to solve this issue?

Success is a callback that happens at a later point in time. So what happens is, the for loop runs 7 times and calls parse 7 times. Then after it has run each of parse success calls will be executed, they look at i which is now at 7.
A simple way to fix this is to wrap the whole thing in an immediate function and create a new closure for i. Something like this
for(var i = 0; i < results.length; i++){
function(iClosure) {
//rest of code goes here, replace i's with iClosure
}(i);
}
Now what will happen is that each success function will have access to it's own iClosure variable and they will be set to the value of i at the point they were created in the loop.

jquery html(array) doesn't insert all items in array

When I run the javascript code below, it load specified amount of images from Flickr.
By var photos = photoGroup.getPhotos(10) code, I get 10 images from cache.
Then, I can see the object has exactly 10 items by checking console.log(photos);
But actual image appeared on the page is less than 10 items...
I have no idea why this work this way..
Thank you in advance.
<html>
<head>
<script src="http://code.jquery.com/jquery-2.1.0.min.js"></script>
<script>
var PhotoGroup = function(nativePhotos, callback) {
var _cache = new Array();
var numberOfPhotosLoaded = 0;
var containerWidth = $("#contents").css('max-width');
var containerHeight = $("#contents").css('max-height');
$(nativePhotos).each(function(key, photo) {
$("<img src='"+"http://farm" + photo["farm"] + ".staticflickr.com/" + photo["server"] + "/" + photo["id"] + "_" + photo["secret"] + "_b.jpg"+"'/>")
.attr("alt", photo['title'])
.attr("data-cycle-title", photo['ownername'])
.load(function() {
if(this.naturalWidth >= this.naturalHeight) {
$(this).attr("width", containerWidth);
} else {
$(this).attr("height", containerHeight);
}
_cache.push(this);
if(nativePhotos.length == ++numberOfPhotosLoaded)
callback();
})
});
var getRandom = function(max) {
return Math.floor((Math.random()*max)+1);
}
this.getPhotos = function(numberOfPhotos) {
var photoPool = new Array();
var maxRandomNumber = _cache.length-1;
while(photoPool.length != numberOfPhotos) {
var index = getRandom(maxRandomNumber);
if($.inArray(_cache[index], photoPool))
photoPool.push(_cache[index]);
}
return photoPool;
}
}
var Contents = function() {
var self = this;
var contentTypes = ["#slideShowWrapper", "#video"];
var switchTo = function(nameOfContent) {
$(contentTypes).each(function(contentType) {
$(contentType).hide();
});
switch(nameOfContent) {
case("EHTV") :
$("#video").show();
break;
case("slideShow") :
$("#slideShowWrapper").show();
break;
default :
break;
}
}
this.startEHTV = function() {
switchTo("EHTV");
document._video = document.getElementById("video");
document._video.addEventListener("loadstart", function() {
document._video.playbackRate = 0.3;
}, false);
document._video.addEventListener("ended", startSlideShow, false);
document._video.play();
}
this.startSlideShow = function() {
switchTo("slideShow");
var photos = photoGroup.getPhotos(10)
console.log(photos);
$('#slideShow').html(photos);
}
var api_key = '6242dcd053cd0ad8d791edd975217606';
var group_id = '2359176#N25';
var flickerAPI = 'http://api.flickr.com/services/rest/?jsoncallback=?';
var photoGroup;
$.getJSON(flickerAPI, {
api_key: api_key,
group_id: group_id,
format: "json",
method: "flickr.groups.pools.getPhotos",
}).done(function(data) {
photoGroup = new PhotoGroup(data['photos']['photo'], self.startSlideShow);
});
}
var contents = new Contents();
</script>
</head>
<body>
<div id="slideShow"></div>
</body>
</html>

I fix your method getRandom() according to this article, and completely re-write method getPhotos():
this.getPhotos = function(numberOfPhotos) {
var available = _cache.length;
if (numberOfPhotos >= available) {
// just clone existing array
return _cache.slice(0);
}
var result = [];
var indices = [];
while (result.length != numberOfPhotos) {
var r = getRandom(available);
if ($.inArray(r, indices) == -1) {
indices.push(r);
result.push(_cache[r]);
}
}
return result;
}
Check full solution here: http://jsfiddle.net/JtDzZ/
But this method still slow, because loop may be quite long to execute due to same random numbers occurred.
If you care about performance, you need to create other stable solution. For ex., randomize only first index of your images sequence.