tldr at the bottom:
I don't really know how to explain my problem so I start with an example.
I have this async function (in reactJS but I think this is a JS related issue).
onUploadDrop = async (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let files = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
var file = await this.getFileByWebkitEntry(item);
files.push(file);
}
console.log(i);
}
// do something with files[]
}
This function is calling another async function:
getFileByWebkitEntry = async (item) => {
return new Promise(resolve => {
item.file(function (file) {
resolve(file);
}, function (err) {
console.log(err);
resolve("");
});
});
}
I'm looping through e.datatransfer.files which are basically some uploaded files or folders. Unfortunately this for-loop gets only executed once.
I did some debugging and found out that if I place a console.log before and after this line: var file = await ... This comes out:
tldr: After the await statement uploadedItems is empty thus ending the loop. Why is this happening?
I solved this by not using async - await but Promises instead.
It looks like this:
onUploadDrop = (e, folderId) => {
e.preventDefault();
// check if the user uploaded files or folders
var uploadedItems = e.dataTransfer.items;
let promises = [];
for (var i = 0; i < uploadedItems.length; i++) {
let item = uploadedItems[i].webkitGetAsEntry();
if (item.isDirectory) {
alert("is directory")
} else {
promises.push(this.getFileByWebkitEntry(item));
}
console.log(i);
}
Promise.all(promises).then(result => {
// do something with result (result = files)
});
Related
I am ordering some items with their priorities. I used a loop for that. However, I get some weird outputs like [1,1,2,2,3] instead of [1,2,3,4,5](these are priorities btw). The loop is below.
const switchPriority = async function(catId, srcI, destI){
let indexofCat;
try {
for (let i = 0; i < data[0].length; i++) {
const element = data[0][i];
if(element.id === catId){
indexofCat = i;
}
}
let Item = Parse.Object.extend('Item')
let itemQuery = new Parse.Query(Item)
for (let i = (srcI>destI?destI:srcI); i < (srcI>destI?(srcI+1):(destI+1)); i++) {
let id = data[1][indexofCat][i].id;
let item = await itemQuery.get(id);
item.set("priority",i+1);
await item.save();
}
} catch (error) {
alert(error)
}
}
Why there is such a problem? When I add alert to the loop, with some delay it gives proper outputs. How can I solve this ? I am appreciate for your help.
I don't know if you forgot, but it's necessary use the word async like this await works. You can insert your code into a function:
async function parse(){
for (let i = (srcI); i < (destI); i++) {
// alert("index in loop "+ i);
let id = data[1][indexofCat][i].id;
let item = await itemQuery.get(id);
// alert(item.get("name"));
item.set("priority",i+1);
await item.save();
}
}
Look at for more details: https://developer.mozilla.org/pt-BR/docs/Web/JavaScript/Reference/Statements/async_function
You can also promisify a function that you needs to guarantee that it be executed before the next instruction. How?
function createPromise (parameter) {
return new Promise((resolve, reject) => {
resolve(console.log(parameter))
})
}
async function test(){
console.log('First')
await createPromise('Promise')
console.log('Second')
}
test()
Look at: https://developer.mozilla.org/pt-BR/docs/Web/JavaScript/Reference/Global_Objects/Promise
I am using chai and javascript to run some automated tests against API endpoints.
In one of my tests, I am pushing results into an array (shown below, the array is named acceptanceCriteriaHashes).
If I put a breakpoint in the before code then I can see the array, and all the elements within the array, are fully populated with the object returned from the db call.
However when I actually come to verify the contents of the array later in the test, the array, although it has the correct number of elements, contains only undefined?
Have tried to push directly with the results of the db call and that doesn't work either?
const getAcceptanceCriteria = (testCase) => {
let getAcceptanceCriteriaResponse;
let myLicenceChecksPassed = false;
const acceptanceCriteriaHashes = new Array();
describeTest(`${testCase.testCaseId} - ${testCase.testDescription}`, async () => {
before(async () => {
if (testCase.myLicenceChecksPassed !== undefined) {
myLicenceChecksPassed = testCase.myLicenceChecksPassed;
}
getAcceptanceCriteriaResponse = await getAcceptanceCriteriaAPICall(testCase.productId, testCase.assetType, testCase.underwriter, myLicenceChecksPassed);
if (getAcceptanceCriteriaResponse != undefined && getAcceptanceCriteriaResponse.body.Detail.length > 0) {
for (let i = 0; i < getAcceptanceCriteriaResponse.body.Detail.length; i++) {
// eslint-disable-next-line prefer-const
let response = await readAcceptanceCriteriaHashes(getAcceptanceCriteriaResponse.body.Detail[i].ContentHash);
if (response != undefined) {
let record = { Hash: response.Hash, Criteria: response.Criteria };
acceptanceCriteriaHashes.push(record);
}
}
console.log(acceptanceCriteriaHashes);
}
});
// other tests successfully run here
it(`${testCase.testCaseId} - Record for contentHash exists in the AcceptanceCriteriaHashes table`, async () => {
expect(acceptanceCriteriaHashes).to.not.be.empty;
expect(getAcceptanceCriteriaResponse.body.Detail.length).to.eql(acceptanceCriteriaHashes.length);
for (let i = 0; i < acceptanceCriteriaHashes.length; i++) {
expect(acceptanceCriteriaHashes[i].to.not.be('undefined', `Returned undefined for ContentHash: ${getAcceptanceCriteriaResponse.body.Detail[i].ContentHash}`));
expect(acceptanceCriteriaHashes[i].Criteria.to.not.be.empty);
}
});
});
};
The issue was I needed to declare the record variable outside of the before code block otherwise, when we exited the before function, it ceased to exist by the time I came to query it in the subsequent test..
Declaration goes just before the before function:
const getAcceptanceCriteria = (testCase) => {
let getAcceptanceCriteriaResponse;
let myLicenceChecksPassed = false;
// eslint-disable-next-line prefer-const
// eslint-disable-next-line no-array-constructor
const acceptanceCriteriaHashes = new Array();
let record;
describeTest(`${testCase.testCaseId} - ${testCase.testDescription}`, async () => {
before(async () => {
if (testCase.myLicenceChecksPassed !== undefined) {
myLicenceChecksPassed = testCase.myLicenceChecksPassed;
}
getAcceptanceCriteriaResponse = await getAcceptanceCriteriaAPICall(testCase.productId, testCase.assetType, testCase.underwriter, myLicenceChecksPassed);
if (getAcceptanceCriteriaResponse != undefined && getAcceptanceCriteriaResponse.body.Detail.length > 0) {
for (let i = 0; i < getAcceptanceCriteriaResponse.body.Detail.length; i++) {
// eslint-disable-next-line prefer-const
let response = await readAcceptanceCriteriaHashes(getAcceptanceCriteriaResponse.body.Detail[i].ContentHash);
if (response != undefined) {
record = { Hash: response.Hash, Criteria: response.Criteria };
acceptanceCriteriaHashes.push(record);
}
}
console.log(acceptanceCriteriaHashes);
}
});
I have a NodeJS script which is making thousands of REST API GET calls to SharePoint to retrieve all folders and files recursively. A problem I have encountered is once in a while I am receiving ECONNRESET errors and 404 Not Found for some folders with unsupported characters. This causes my code to stop and I lose my progress in retrieving all the files and folders. Is there something I can do to prevent these errors from stopping my code and just have it ignore the error and continue onto the next folder?
I thought catching the error and console logging it would suffice but this still ends the execution of the code. Thanks
//Imports and Declarations
const prompt = require("prompt-sync")({ sigint: true });
const { getCreds, pullFromSharePoint } = require('./helper');
const sprequest = require('sp-request');
var path = require('path');
let creds = getCreds();
let spr = sprequest.create(creds)
let SPUrl = 'url to sharepoint site';
let files = [];
let folders = [];
let scannedFolders = [];
let reiteration = false;
let processingReq = false;
//Prompt user for which folder to copy
let targetFolder = prompt("Folder to copy: ");
//Call function to get list of files and folders in specified folder
// getFolderContents(targetFolder);
processInfo([], targetFolder)
//Gets list of files/folders inside folder
function getFolderContents(targetFolder) {
return new Promise((resolve, reject) => {
logger.info('Scanning this folder: ' + targetFolder)
//Skip if folder name contains chars we dont support
if (targetFolder.includes("'")) {
targetFolder = targetFolder.replace(/'/g, "''")
}
//If reiteration = true, format the target folder
if (reiteration === true) {
targetFolder = targetFolder.replace('/sites/sharepointsite', '');
}
console.log('Scanning this folder: ' + targetFolder)
targetFolder = encodeURIComponent(targetFolder)
var url = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')/Files?$filter=TimeLastModified gt datetime'2021-07-01T00:00:00'`;
var url2 = `${SPUrl}/_api/web/GetFolderByServerRelativeUrl('Shared%20Documents/${targetFolder}')?$expand=Folders`;
//THESE SPR CALLS ARE THE REST API CALLS TO SHAREPOINT RESULTING IN THE MOST 404 ERRORS, HOW TO IGNORE ERRORS FROM THESE?
// Call to get list of files
spr.get(url).then(response => {
console.log('Calling SP API to get files')
//Adds files to array
let fileRes = response.body.d.results;
for (let i = 0; i < fileRes.length; i++) {
files.push(fileRes[i].ServerRelativeUrl)
}
console.log(files.length)
}).catch(function (err) {
console.log('Fetch Error :-S', err);
});
//Call to get list of folders
spr.get(url2).then(response => {
//Adds folders to array
let folderRes = response.body.d.Folders.results;
for (let j = 0; j < folderRes.length; j++) {
folders.push(folderRes[j].ServerRelativeUrl)
}
//Push current folder read through to another array so we dont scan it multiple times
scannedFolders.push('/sites/sharepointsite/Shared Documents' + targetFolder);
resolve(folders);
}).catch(function (err) {
console.log('Fetch Error :-S', err);
});
})
}
//If folders exist in folders array scan them; once all folders have been scanned, send files to be copied
async function processInfo(folders, firstFolder) {
let firstRun = await getFolderContents(firstFolder);
let firstGone = false;
if (firstRun) {
if (firstRun.length > 0) {
for (let k = 0; k < firstRun.length; k++) {
await sleep(500);
//If folder has already been scanned, remove from array and skip to next iteration
if (scannedFolders.includes(firstRun[k])) {
if (k === 0) {
firstRun.splice(k, 1)
}
firstRun.splice(k, k)
continue;
}
//Else if folder has not been scanned, send it to be scanned
else {
if (firstRun[k].includes('%') || firstRun[k].includes('%')) {
console.log('skipping')
continue;
}
reiteration = true;
let foldersScanned = await getFolderContents(firstRun[k]).catch(function (err) {
console.log('Fetch Error :-S', err);
});;
//Send each file to pullFile() function to be downloaded
// if (foldersScanned && k == firstRun.length - 1) {
if (firstRun[k] == 10) {
do {
await pullFile(files[0]);
files.shift();
} while (files.length > 0)
}
}
}
}
}
console.log(files.length)
}
//Manipulate file string to get SP folder and File Name, then call the helper function to download the file
async function pullFile(file) {
let filename = file.replace(/^.*[\\\/]/, '')
let spFolder = file.replace('/sites/sharepointsite/', '')
spFolder = spFolder.replace(filename, '');
let downloadPath = path.join('./testfolder', spFolder)
const sppullContext = {
siteUrl: SPUrl, //SharePoint URL
creds: creds //Credentials
};
const sppullOptions = {
spRootFolder: spFolder, // The folder path for the file in SharePoint
dlRootFolder: downloadPath, // Where the file is saved locally
strictObjects: [filename], // Only download the filename specified as a query parameter
muteConsole: true
};
pullFromSharePoint(sppullContext, sppullOptions) //Pull file with filename from SharePoint
.then(() => {
return true;
}).catch(err => {
return err;
});
}
//Timer to wait specified amount of time
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
Code from helper.js
//Imports and Declarations
var fs = require('fs');
const { SPPull } = require('sppull');
const winston = require('winston');
//Winston logger
const logger = winston.createLogger({
level: 'info',
format: winston.format.json(),
defaultMeta: { service: 'user-service' },
transports: [
new winston.transports.File({ filename: 'error.log', level: 'error' }),
new winston.transports.File({ filename: 'info.log' }),
],
});
/**
* Returns the SharePoint credentials object to use.
*/
function getCreds() {
return {
username: 'UN',
password: 'PW'
};
}
module.exports.getCreds = getCreds;
/**
* Pulls a file from SharePoint and then checks for errors.
*
* #param sppullContext context object for sppull
* #param sppullOptions options object for sppull
*/
async function pullFromSharePoint(sppullContext, sppullOptions) {
SPPull.download(sppullContext, sppullOptions)
.then((res) => {
logger.info(res)
return res
})
.catch((err) => {
logger.error(err)
return err
});
}
My objective is to read data from two files and compare the data. My input files are result3.json and result4.json.The data in these files are coma separated.
result3.json
[
"temp1.txt",
"temp2.txt",
]
node:
function readFromFile(file) {
var fileNames = [];
//setTimeout(function() {
fs.readFile(file,function(err, data){
if (err) {
return console.log(err);
}
var temp = JSON.parse(data.toString().split(","));
// console.log(temp.length);
for (let index = 0; index < temp.length; index++) {
//console.log(temp[index]);
fileNames.push(temp[index]);
//console.log(fileNames[index]);
}
Done(); // to block the async call
});
//},3000);
//console.log(fileNames.length);
return fileNames;
}
var baseListOfFiles = readFromFile('./output/result3.json'); // Assume this is the base file
var currentListOfFiles = readFromFile('./output/result4.json'); // Assume this is the current file
function Done(){
//console.log('Out baseListOfFiles + ' + baseListOfFiles.length);
for (let index = 0; index < baseListOfFiles.length; index++) {
console.log("[baseListOfFiles] " + baseListOfFiles[index]);
}
//console.log('Out currentListOfFiles+ ' + currentListOfFiles.length);
for (let index = 0; index < currentListOfFiles.length; index++) {
console.log("[currentListOfFiles] " + currentListOfFiles[index]);
}
}
Above is my code. It seems to be async call, so it always return 0 fileNames.
Is there any way to control it?
Here's example code using Promises:
const fs = require('fs');
function readFromFile(file) {
return new Promise((resolve, reject) => {
fs.readFile(file, function (err, data) {
if (err) {
console.log(err);
reject(err);
}
else {
resolve(JSON.parse(data));
}
});
});
}
const promises = [
readFromFile('./output/result3.json'),
readFromFile('./output/result4.json')
];
Promise.all(promises).then(result => {
console.log(result);
baseListOfFiles = result[0];
currentListOfFiles = result[1];
// do more stuff
});
First, an array promises is built; each Promise reads the file, then calls resolve with the result.
This array is passed to Promise.all(), which then calls the callback, passing the array of results in the same order.
You're right, readFile is async. What you're looking for is readFileSync: https://nodejs.org/api/fs.html#fs_fs_readfilesync_path_options
With that can can do:
const data = fs.readFileSync(file);
//do something with data
There are a few ways to 'promisify' readFile if you like, the options are discussed here: Using filesystem in node.js with async / await
I was practicing Express 4.x and noticed the following:
app.get('/fake', function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
jsf.resolve(fakeSchema).then(function(iter) {
obj.push(iter);
});
}
res.send(obj);
});
So, going to that route, I get "[ ]", while I was expecting to receive an array of 3 (fake) documents.
FYI, when logging each loop, I can clearly see the documents generated, even inside the array.
Any explanation?
Your jsf.resolve functiion is async so you can use async/await for this to perform task in sync manner.
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function
app.get('/fake', async function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
try {
var iter = await jsf.resolve(fakeSchema);
obj.push(iter);
} catch (e) {}
}
res.send(obj);
});
Although #Nishant's provided answer works, I suggest using this approach.
let jsf = {};
// faking your jsf.resolve method
jsf.resolve = (param) => {
return new Promise((resolve) => {
setTimeout(() => {
resolve(Math.random());
}, 1000);
})
};
let fakeSchema = {};
let obj = [];
let promises = [];
for (let i = 0; i !== 3; i++) {
promises.push(jsf.resolve(fakeSchema).then(function (iter) {
obj.push(iter);
}));
}
Promise.all(promises).then(() => {
console.log(obj);
});
This allows all the promises to run concurrently, imagine your jsx.resolve takes a long time to complete, using await would freeze your entire appp.
As opposed to this. Note the runtime.
(async () => {
let jsf = {};
// faking your jsf.resolve method
jsf.resolve = (param) => {
return new Promise((resolve) => {
setTimeout(() => {
resolve(Math.random());
}, 1000);
})
};
let fakeSchema = {};
let obj = [];
for (let i = 0; i !== 3; i++) {
obj.push(await jsf.resolve(fakeSchema));
}
console.log(obj);
})();
#Nishant Dixit's answer also correct!
You can try this simple solution also, if you like :
app.get('/fake', function(req, res) {
var obj = [];
for (let i = 0; i < 3; i++) {
try {
jsf.resolve(fakeSchema).then(function(iter) {
obj.push(iter);
res.send(obj);
} catch (e) {
res.send(e);
}
});
};
});