We Keep Coding

How to send multiple different requests using promises, and have all run one after the other?

I have this script that I use to import some data from an API, and into my database. Since this process is very time consuming, it often times out because on some of the items processed there is a lot of data to process..
I came with this solution a while ago, using promises, to first do the request to the API, then after it finishes I would prepare the data and put it into a temporary csv file, then I would fire another request to split that file into multiple smaller files, then... you get the idea... it was working, but I need to add to it some extra requests, I just can't make it work... I probably just need to simplify my logic.
Anyone can help me improve this code to make it easier to add those extra requests and keep it sending one request after the other?
This is the (over simplified) script in question:
window.importTrialsScripts = {};
window.addEventListener('DOMContentLoaded', function() {
importTrialsScripts.app.initialize();
});
(function(importTrialsScripts, document, $) {
importTrialsScripts = importTrialsScripts || {};
const app = {
ajaxurl: 'myajaxurl',
initialize: function() {
this.submitHandler();
},
submitHandler: function() {
const self = this;
document.querySelector('#start-import').addEventListener('click', function() {
self.pullTrialsFromApi();
});
},
pullTrialsFromApi: function() {
let data = new FormData();
data.append('action', 'pull_trials_from_api');
[123, 456, 789].forEach(function(str) {
data.append('ids[]', str);
});
this.startPullingTrials(data);
},
startPullingTrials: function(data) {
const self = this;
let promise = new Promise(function(resolve, reject) {
self.sendAjaxRequest(data, function() {
if (this.status === 200) {
const response = JSON.parse(this.response);
if (! response.success) {
alert('The API could not be reached. Please try again.');
console.error('Error!!', response);
return;
}
resolve(response.data);
}
else {
console.error('there was an error in the request', this);
reject(this);
}
});
});
promise.then(function(chunks) {
const processingChunks = Object.values(chunks).map(function(chunk) {
return self.processChunk(chunk);
});
Promise.all(processingChunks).then(function (processedTrials) {
console.log('finished', processedTrials);
});
}, function(err) {
console.error('promise rejected', err);
});
},
processChunk: function(chunkTrials) {
const self = this;
let data = new FormData();
data.append('action', 'process_trials_chunk');
Object.values(chunkTrials).forEach(function(chunk) {
data.append('chunk[]', JSON.stringify(chunk));
});
return new Promise(function(resolve, reject) {
self.sendAjaxRequest(data, function() {
if (this.status === 200) {
const response = JSON.parse(this.response);
if (! response.success) {
console.error('Error!!', response.data);
return;
}
resolve(response.data);
}
else {
console.log('there was an error in the request', this);
reject(this);
}
});
});
},
splitToMultipleFiles: function() {
const self = this;
const data = new FormData();
data.append('action', 'split_location_files');
return new Promise(function(resolve, reject) {
self.sendAjaxRequest(data, function() {
if (this.status === 200) {
const response = JSON.parse(this.response);
if ( ! response.success ) {
console.error('Error!!', response.data);
return;
}
resolve(response.data.files);
}
else {
console.log('there was an error in the request', this);
reject(this);
}
});
});
},
processLocation: function(file) {
const self = this;
let data = new FormData();
data.append('action', 'process_location_data');
data.append('file', file);
return new Promise(function(resolve, reject) {
self.sendAjaxRequest(data, function() {
if ( this.status === 200 ) {
const response = JSON.parse(this.response);
if (! response.success) {
console.error('Error!!', response.data);
return;
}
resolve(response.data);
}
else {
console.log('there was an error in the request', this);
reject(this);
}
});
});
},
sendAjaxRequest: function(data, callback) {
const self = this;
let xhr = new XMLHttpRequest();
xhr.open('POST', ajaxurl);
xhr.onload = callback;
xhr.addEventListener('timeout', function(e) {
console.error('the request has timed out', e);
});
xhr.addEventListener('error', function(e) {
console.error('the request returned an error', e);
});
xhr.addEventListener('abort', function(e) {
console.error('the request was aborted', e);
});
xhr.send(data);
},
};
$.extend(importTrialsScripts, {
app: app
});
}).apply(this, [window.importTrialsScripts, document, jQuery]);

PHP backend with JS xHTMLRequest frontend

In JS, I wanted to create a function that made a xHTMLRequest to a backend PHP server, problem is I want JS to wait for the response, otherwise it will display 'undefined'.
function xhrReq(method, args) {
let xhr = new XMLHttpRequest();
xhr.open(method, 'http://localhost/example/php/example.php');
xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
xhr.send(args);
xhr.onreadystatechange = ()=> {
if(xhr.readyState == 4) {
return xhr.response;
}
}
How can I make this function return the response value?

You can use fetch in a async function:
(async () => {
try {
//const args = ...;
var headers = new Headers();
headers.append("Content-Type", "application/x-www-form-urlencoded");
const response = await fetch('http://localhost/example/php/example.php', {
method: 'POST', // or other
headers,
body: args
});
} catch (err) {
//process error
}
})()
or you can promisify your function :
function xhrReq(method, args) {
return new Promise(function(resolve, reject) {
var xhr = new XMLHttpRequest();
xhr.open(method, 'http://localhost/example/php/example.php');
xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
xhr.onload = function() {
if (xhr.status === 200) {
resolve(xhr.response);
} else {
reject(Error(`XHR request failed. Error code: ${xhr.statusText}`));
}
};
xhr.onerror = function() {
reject(Error('There was a network error.'));
};
xhr.send(args);
});
}
And use it in a async function (or use promise) to get the response.

Change browser url after post request in node server

I am implementing a sign up page in which I want to change the browser url after submitting the form and sending post request. Indeed I want the server to serve another static file after receiving post request. How can I do that?
Here is my xhr request in client side using vanilla js:
function signup(e) {
var data = {
name: _elements.fullname.value,
username: _elements.username.value,
password: _elements.password.value
};
data = JSON.stringify(data);
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if(xhr.readyState === 4 && xhr.status === 200) alert('Signed in successfully!');
}
xhr.open('POST', 'signup', true);
xhr.setRequestHeader('Content-Type', 'text/plain')
xhr.send(data);
}
And This is my server.js file:
const http = require('http');
const requestHandler = require('./req-handler').requestHandler;
http.createServer(requestHandler).listen(8080);
req-handler.js:
if (req.method === 'GET') {
switch (req.url) {
case '/':
routeHandler = rootHandler;
break;
}
} else if (req.method === 'POST') {
switch (req.url) {
case '/signup':
routeHandler = signupHandler;
break;
}
}
if (!routeHandler) {
routeHandler = staticFileHandler;
}
routeHandler(req, res);
function rootHandler(req, res) {
req.url = 'signup.html';
staticFileHandler(req, res);
}
function signupHandler(req, res) {
req.url = 'index.html';
var jsonData = '';
req.on('data', data => {
jsonData += data.toString('utf-8')
});
req.on('end', () => {
staticFileHandler(req, res);
});
}
function staticFileHandler(req, res) {
console.log(req.url)
fs.readFile('client/' + req.url, function (err, data) {
if (err) {
res.writeHead(500);
res.write(err.name);
res.end();
}
console.log(data)
res.writeHead(200);
res.write(data);
res.end();
});
}
All the static files (.html and .css) are in /client folder. By the way, I do not want to use any library/framework.

You can navigate to a new page, using vanilla js.
xhr.onreadystatechange = function() {
if(xhr.readyState === 4 && xhr.status === 200) {
alert('Signed in successfully!');
document.location.href = {Your_new_URL}
}
}
If a file path can vary - You can always specify it in the response JSON on the server and use it in XHR success-callback.

GET request from browser works to download file to local but XMLHttpRequest Javascript script does not download file

I'm having trouble with XMLHttpRequest I think, when I navigate to localhost/dashboard/downloadfile?file-name=hw3.txt the file downloads locally but If I use the function checkDownload() to start an XMLHttpRequest the file does not get downloaded.
Here is my client code:
function checkDownload() {
const filename = "hw3.txt";
const xhr = new XMLHttpRequest();
xhr.responseType = "blob";
xhr.open('GET', `/dashboard/downloadfile?file-name=${ filename }`);
xhr.onreadystatechange = () => {
if(xhr.readyState === 4) {
if(xhr.status === 200) {
}
}
}
xhr.send();
}
And then here is my server code:
app.get('/dashboard/downloadfile', requiresLogin, (req, res) => {
const userid = req.user.id;
const filename = req.query['file-name'];
db.getFileKey([userid, filename], (keyres) => {
const params = {
Bucket: S3_BUCKET,
Key: keyres.rows[0].filekey,
};
res.setHeader('Content-disposition', `attachment; filename=${ filename }`);
res.setHeader('Content-type', `${ mime.getType(keyres.rows[0].filetype) }`);
s3.getObject(params, (awserr, awsres) => {
if(awserr) console.log(awserr);
else console.log(awsres);
}).createReadStream().pipe(res);
});
});

I got it working. Instead of trying to create a read stream from s3.getObject() I generated a signed url to the s3 object on the server and returned that to the client, then used an 'a' html element with element.href = signedRequest and used javascript to click that element. The new problem I'm running into is that I can't figure out a way to set the metadata for the s3 object when it is initially uploaded, I needed to manually change the metadata on an individual s3 object through the aws console so that it had the header Content-Disposition: attachment; filename=${ filename }.
changed client code:
function initDownload(filename) {
const xhr = new XMLHttpRequest();
xhr.open('GET', `/sign-s3-get-request?file-name=${ filename }`);
xhr.onreadystatechange = () => {
if(xhr.readyState === 4) {
if(xhr.status === 200) {
const response = JSON.parse(xhr.responseText);
startDownload(response.signedRequest, response.url);
}
}
}
xhr.send();
}
function startDownload(signedRequest, url) {
var link = document.createElement('a');
link.href = signedRequest;
link.setAttribute('download', 'download');
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
}
changed server code:
app.get('/sign-s3-get-request', requiresLogin, (req, res) => {
const userid = req.user.id;
const filename = req.query['file-name'];
db.getFileKey([userid, filename], (keyres) => {
const s3Params = {
Bucket: S3_BUCKET,
Key: keyres.rows[0].filekey,
Expires: 60,
};
s3.getSignedUrl('getObject', s3Params, (err, data) => {
if (err) {
// eslint-disable-next-line
console.log(err);
res.end();
}
const returnData = {
signedRequest: data,
url: `https://${S3_BUCKET}.s3.amazonaws.com/${ keyres.rows[0].filekey }`,
};
res.write(JSON.stringify(returnData));
res.end();
});
});
});

You are getting a blob back from the server, so in order to download you need to do something when xhr.status === 200.
Something like this:
...
if(xhr.status === 200) {
var fileUrl = URL.createObjectURL(xhr.responseText)
window.location.replace(fileUrl)
}
...

To download having the URL you could use the attribute download of a tag:
<a download="something.txt" href="https://google.com">Download Google</a>
If you use xhr.responseType = "blob", you have to do somethig like:
function checkDownload() {
const filename = "hw3.txt";
const xhr = new XMLHttpRequest();
xhr.responseType = "blob";
xhr.open('GET', 'https://jsonplaceholder.typicode.com/todos/1');
xhr.onreadystatechange = () => {
if(xhr.readyState === 4) {
if(xhr.status === 200) {
var reader = new FileReader();
reader.readAsArrayBuffer(xhr.response);
reader.addEventListener("loadend", function() {
var a = new Int8Array(reader.result);
console.log(JSON.stringify(a, null, ' '));
});
}
}
}
xhr.send();
}
checkDownload()
But that code doesn't download the file.

Cordova file upload

I want to upload a video to YouTube from my phone device storage. However when I upload the file it comes through as blank. When I use the same upload code but with a web file, it works. Wondering where I am going wrong!
Method one
everything uploads correctly and the video plays on YouTube.
loadWebFile('assets/intro.mpg');
function loadWebFile(url) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url);
xhr.responseType = 'blob';
xhr.onload = function (e) {
uploadFile(xhr.response); // type: Blob
};
xhr.onerror = function (e) {
console.log('loadWebFile.onerror', e);
};
xhr.send();
};
Method two
The video title and description appears on YouTube, but the video is blank. I'm definitely passing through a valid file.
window.requestFileSystem = window.requestFileSystem || window.webkitRequestFileSystem;
if (window.webkitStorageInfo) {
window.webkitStorageInfo.requestQuota(access, 1024 * 1024, function (bytes) {
if (window.requestFileSystem) {
window.requestFileSystem(access, bytes, function (filesystem) {
loadFile('/Movies/intro.mpg');
}, me.onError);
} else {
window.alert('requestFileSystem not supported');
}
}, me.onError);
} else {
window.alert('webkitStorageInfo not supported');
}
// this sends an empty video to YouTube
function loadFile(path) {
filesystem.root.getFile(path, null, function (fileEntry) {
fileEntry.file(function (file) {
uploadFile(file); // type: File
});
}, function (e) {
console.log('loadFile.error', e);
});
}
Both methods share the same upload function:
// uploads using the YouTube script
// https://github.com/youtube/api-samples/blob/master/javascript/cors_upload.js
function uploadFile(file) {
var metadata = {
snippet: {
title: 'Video title',
description: 'Video description',
tags: 'Video tags',
categoryId: 22
},
status: {
privacyStatus: 'unlisted'
}
};
var uploader = new MediaUploader({
baseUrl: 'https://www.googleapis.com/upload/youtube/v3/videos',
file: file,
token: accessToken,
metadata: metadata,
params: {
part: Object.keys(metadata).join(',')
},
onError: function (e) {
console.log('onError', JSON.parse(e));
},
onProgress: function (e) {
console.log('onProgress', e);
},
onComplete: function (e) {
console.log('onComplete', JSON.parse(e));
}
});
uploader.upload();
};
I've have an example project with some of the code (minus the upload script) here:
https://github.com/kmturley/cordova-files

Here another working solution. I tested right now and it does work. You need standard cordova-plugin-file
function uploadFileToServer (fileUri, fileName, remoteUrl, callback) {
window.resolveLocalFileSystemURL(fileUri, function (fileEntry) {
fileEntry.file(function (file) {
var reader = new FileReader()
reader.onloadend = function () {
var blob = new Blob([new Uint8Array(this.result)], { type: 'application/octet-stream' })
var fd = new FormData()
fd.append('file', blob, fileName)
var xhr = new XMLHttpRequest()
xhr.open('POST', remoteUrl, true)
xhr.onload = function () {
if (xhr.status === 200) {
if (typeof callback === 'function') { callback() }
} else {
if (typeof callback === 'function') { callback(xhr.status) }
}
}
xhr.onerror = function (err) {
if (typeof callback === 'function') { callback(err) }
}
xhr.send(fd)
}
reader.readAsArrayBuffer(file)
}, function (err) {
if (typeof callback === 'function') { callback(err) }
})
})
}
You call it:
uploadFileToServer('file:///storage/emulated/0/Android/data/myfile.jpg',
'myfile.jpg',
'https://example.com/upload_url',
(err) => {
if (err) {
console.error('Error uploading file', err)
} else {
console.log('Upload done it with success')
}
})

The oficial file-transfer plugin is deprecated and when I'm writing this, its test script fails.
This made me use a pure javascript approach and it does work
function uploadFile (localPath, fileName, remoteUrl, callback) {
// loads local file with http GET request
var xhrLocal = new XMLHttpRequest()
xhrLocal.open('get', localPath)
xhrLocal.responseType = 'blob'
xhrLocal.onerror = () => {
callback(Error('An error ocurred getting localpath on' + localPath))
}
xhrLocal.onload = () => {
// when data is loaded creates a file reader to read data
var fr = new FileReader()
fr.onload = function (e) {
// fetch the data and accept the blob
console.log(e)
fetch(e.target.result)
.then(res => res.blob())
.then((res) => {
// now creates another http post request to upload the file
var formData = new FormData()
formData.append('imagefile', res, fileName)
// post form data
const xhrRemote = new XMLHttpRequest()
xhrRemote.responseType = 'json'
// log response
xhrRemote.onerror = () => {
callback(Error('An error ocurred uploading the file to ' + remoteUrl))
}
xhrRemote.onload = () => {
if (typeof callback === 'function') {
callback(null, 'File uploaded successful, ' + xhrRemote.response)
}
}
// create and send the reqeust
xhrRemote.open('POST', remoteUrl)
xhrRemote.send(formData)
})
}
fr.readAsDataURL(xhrLocal.response) // async call
}
xhrLocal.send()
}
Now just call it something like this
uploadFile('file:///storage/emulated/0/Android/data/myfile.jpg',
'myfile.jpg',
'https://example.com/upload_url',
(err, res) => {
if (err) {
console.error(err)
} else {
console.log(res)
}
})

So to upload files I realised for:
web files use a CORS upload:
https://github.com/youtube/api-samples/blob/master/javascript/cors_upload.js
local device files us the cordova file transfer plugin:
https://github.com/apache/cordova-plugin-file-transfer
The code i'm now using for local files which uploads the file and sets correct metadata:
function uploadVideo(fileURL) {
var options = new FileUploadOptions();
options.fileKey = 'file';
options.fileName = fileURL.substr(fileURL.lastIndexOf('/') + 1);
options.mimeType = 'video/mpg';
options.chunkedMode = false;
options.headers = {
Authorization: 'Bearer ' + accessToken
};
options.params = {
"": {
snippet: {
title: 'Video title',
description: 'Video description',
tags: 'Video tags',
categoryId: 22
},
status: {
privacyStatus: 'unlisted'
}
}
};
var ft = new FileTransfer();
ft.upload(fileURL, 'https://www.googleapis.com/upload/youtube/v3/videos?part=snippet,status', function (data) {
console.log('upload success', data);
}, function (e) {
console.log('upload error', e);
}, options, true);
ft.onprogress = function (progressEvent) {
console.log('onprogress: ' + ((progressEvent.loaded / progressEvent.total) * 100) + '%');
};
}
And I also had to modify the plugin to allow the metadata to be passed through to YouTube using a single request:
FileTransfer.java
lines 374 - 376
beforeData.append("Content-Disposition: form-data; name=\"").append(key.toString()).append("\";");
beforeData.append(" filename=\"").append("file.json").append('"').append(LINE_END);
beforeData.append("Content-Type: ").append("application/json").append(LINE_END).append(LINE_END);
If you do modify the plugin, remember cordova caches this code. I use this command to force it to update the plugin:
cordova platform remove android; cordova platform add android;