scan duplicates item with DynamoDB - javascript

I'd like to scan items and avoid to use duplicates code.
so, I am trying to use for-of asynchronously for it.
async function checkDupl(){
const arr = new Array(10).fill(0);
let code = '';
for(const i of arr){
//generate RANDOM CODE
//for example, it would be '000001' to '000010'
code = (Math.floor(Math.random() * 10) + 1).toString().padStart(6,"0");
const params = { ... }; // it has filterExpression the code I generated randomly
await DYNAMO_DB.scan(params, (err, res) => {
if(res.Items.length === 0) {
/* no duplicate! */
return code;
}
});
}
return code;
}
console.log(checkDupl());
// it always return '';
What I have missed or misunderstood?

await just waits a Promise (or thenable object), but you are using await with a "void" function (You use DYNAMO_DB.scan as a callback styte function).
My suggestion, Use DYNAMO_DB.scan with Promise style (The way)
async function checkDupl() {
const arr = new Array(10).fill(0);
let code = '';
for (const i of arr) {
//generate RANDOM CODE
//for example, it would be '000001' to '000010'
code = (Math.floor(Math.random() * 10) + 1).toString().padStart(6, "0");
const params = { ... }; // it has filterExpression the code I generated randomly
const res = await DYNAMO_DB.scan(params).promise(); // convert to promise
if (res.Items.length === 0) {
/* no duplicate! */
return code;
}
return code;
}
}
(async () => {
console.log(await checkDupl());
})();

Related

(CLOSED) Return statement never gets called. NodeJS loop is still finite and doesn't go forever. What's happening?

I was able to fix this by changing the contents of the .push() of allTeams (second to last instance of .push() in the code)
The final return statement should return an array of objects. (allObjectTeams, right after await browser.close()), BUT it returns undefined
Placing the variable right above the return statement still does not solve the issue.
Commenting the biggest loop (the one that iterates over blocks.length) makes that final return statement work but I don't know what part of said loop is messing it up, Commenting out anything that does an action to this said variable doesn't solve the issue
makeTeam() is an instance of recursion but it looks fine to me. Essentially taking a random set of elements from the people array and placing it as a property of an object, then to add than object to an array,
The function also removes the specific elements that were chosen from that array so if there are still more, It calls itself again. A heads-up but I don’t think it does anything.
I am using Puppeteer.
Code (JS):
const start = async () => {
let teamSize = 2
let allObjectTeams = [] //This is the variable I am trying to return
let allTeams = []
const browser = await require("puppeteer").launch()
const page = await browser.newPage()
await page.goto("https://www.when2meet.com/?16521246-5ySm2")
const blocks = await page.$$("#GroupGridSlots > div")
for (let i = 0; i < blocks.length; i++) { // This is the loop I'm talking about, blocks.length is finite. This loop does not go on forever for the return statement never to be called.
await blocks[i].hover()
const team = await page.$eval("#Available", ((availableList, teamSize) => {
const people = availableList.innerHTML.split("<br>")
people.pop()
function getMultipleRandom(arr, num) { // Grabs random elements from array
const shuffled = [...arr].sort(() => 0.5 - Math.random());
return shuffled.slice(0, num);
}
if (people.length == teamSize) {
const availableTeam = {}
availableTeam.people = people
return availableTeam
} else if (people.length > teamSize) {
const teamArray = []
const makeTeam = (currentLength) => {
const availableTeam = {}
const teamPeople = getMultipleRandom(people, teamSize)
availableTeam.people = teamPeople
teamArray.push(availableTeam)
if ((currentLength - teamSize) > teamSize) {
teamArray[teamArray.length - 1].people.forEach(person => {
people.forEach(availablePerson => {
if (availablePerson == person) {
people.splice(people.indexOf(person), 1)
}
})
})
makeTeam(currentLength - teamSize)
}
}
makeTeam(people.length)
return teamArray // returns array of Objects, But THIS is not the one
} else {
return
}
}), teamSize)
if (!team) {
} else if (!Array.isArray(team) && allTeams.includes(team.people.toString())) {
return
} else if (Array.isArray(team)) {
team.forEach(t => {
allTeams.forEach(registeredTeam => {
if (t == registeredTeam) {
team.splice(team.indexOf(t), 1)
}
})
})
for (let i = 0; i < team.length; i++) {
allTeams.push(team[i].toString())
const availableDate = await page.$eval("#AvailableDate", (availableDate => {
return availableDate.textContent.split("Sunday ").pop()
}))
team[i].time = availableDate
allObjectTeams.push(team[i])
}
} else if (team.hasOwnProperty("people")) {
allTeams.push(team.people.toString()) // this was changed to allTeams.push(team.toString())
const availableDate = await page.$eval("#AvailableDate", (availableDate => {
return availableDate.textContent.split("Sunday ").pop()
}))
team.time = availableDate
allObjectTeams.push(team)
}
}
await browser.close()
return allObjectTeams // This return statement returns undefined. THE OTHERS DO NOT
}
start().then((arr) => {
console.log(arr)
})

My Asynchronous Loop returns an array of unknown Numbers

I am trying to scrape the list of results from Google Maps.
Example: Visit https://www.google.com/maps/search/gym+in+nyc
Get all results in an array, loop Starts and click element#1, extract data, back to results page and continue loop.
const finalData = async function () {
const arr = [];
const resultList = [...[...document.querySelectorAll("[aria-
label^='Results for']")][0].children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
let response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
async function scrapePage(elem) {
// Clicks each element
let click = await elem.click();
// Grabs Just the Title
const titleText = await setTimeout(function () {
let title = document.querySelector(".section-hero-header-title
span").innerText;
return title;
}, 3000);
// setTimeout to cause delay before click the back button
setTimeout(function () {
document.querySelector(".section-back-to-list-button").click();
}, 5000);
return titleText;
}
const final = finalData().then((value) => {
return value;
});
I have no idea why when I try the above code in devtools, only the last result is clicked and why my const variable "final" is filled with array of random numbers.
The problem is that you had assumed that the await operator worked on setTimeout; setTimout, setInterval and related functions do not use promises at all.
When working with time orientated code, generally I would set up a helper function that uses a promise:
const delay = seconds => new Promise(
resolve => {
setTimeout(resolve, seconds);
}
);
Updating your code to wait for the timeouts from here is simple:
const finalData = async () => {
const arr = [];
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
const response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
async function scrapePage(elem) {
// Clicks each element
const click = await elem.click(); // this is not asynchronous! click also returns undefined
// Grabs Just the Title
await delay(3);
const titleText = document.querySelector(".section-hero-header-title span").innerText;
// delay before click[ing] the back button
await delay(5);
document.querySelector(".section-back-to-list-button").click();
return titleText;
}
const final = finalData();
And, on a related note, this piece of code waits for every single result in sequence, this takes at least 8 seconds per iteration:
const finalData = async () => {
const arr = [];
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((even, i) => !(i % 2));
for (const eachElement of resultList) {
let response = await scrapePage(eachElement);
arr.push(response);
}
return arr;
};
If you wanted to concurrently execute every iteration, you may want to consider mapping the function across the array and using Promise.all, like so:
const finalData = async () => {
const resultList = [...document.querySelector("[aria-label^='Results for']").children].filter((_, i) => !(i % 2));
return Promise.all(resultList.map(scrapePage));
};
You can try this:
async function scrapePage(elem) {
// Clicks each element
let click = await elem.click();
// Grabs Just the Title
const titleText = await new Promise((resolve) => {
setTimeout(function () {
let title = document.querySelector(".section-hero-header-title span").innerText;
resolve(title);
}, 3000);
})
// setTimeout to cause delay before click the back button
setTimeout(function () {
document.querySelector(".section-back-to-list-button").click();
}, 5000);
return titleText;
}

Recursive async function with promise resolve

I have a recursive async function and i can get expected output via fs.writeFile where return new Promise.. is, but cannot return asynchronously. I got undefined. How can I fix that without disrupt my code structure, thanks.
async function get_followings(all_data = [], cursor, count = 0) {
if (count == 2) {
return new Promise(resolve => {
resolve(all_data.length)
})
}
var followings = await client.getFollowings({ userId: 2896863162, first: 50, after: cursor })
var total = followings.count
var cursor = followings.page_info.end_cursor
var data = followings.data
all_data = all_data.concat(data)
await get_followings(all_data, cursor, count + 1)
}
console.log(await get_followings())

How to guarantee sequential order with angular http rest api in for loop?

I'm trying to create a form that allows you to create multiple resources in sequential order.
Example below
Floor 1
Floor 2
Floor 3
...
Floor 9
The problem with the code is that the order is not guarantee.
My code below
let startAt = this.addAreasForm.controls['startAt'].value
const name = this.addAreasForm.controls['name'].value
const newArea = {name: name}
for (let i = 1; i < (amount + 1); i++) {
newArea.name = name + ' ' + startAt
startAt++
this.areasService.createArea(newArea, parentId)
.subscribe(
area => this.added.emit(area)
)
}
Can come back like
Floor 2
Floor 3
Floor 1
Floor 5
Floor 4
How do you handle async api calls to guarantee sequential order?
You can use async / await for that purpose with the Promise resolve:
for (let i = 1; i < (amount + 1); i++) {
await new Promise(resolve => {
newArea.name = name + ' ' + startAt
startAt++
this.areasService.createArea(newArea, parentId)
.subscribe(
area => {
this.added.emit(area);
resolve();
});
});
}
Remember to put async before your function. See this demo on StackBlitz.
You can try something like this, I don't exactly all your code from your services, but the main idea is this: In order to execute async code in order, you can build an array of promises and then to use Promise.all to take each result in the same order from the creation:
Promise.all
let startAt = this.addAreasForm.controls['startAt'].value;
const name = this.addAreasForm.controls['name'].value;
const newArea = {name: name};
Keep your services into variables I don't know from where your context comes.
const areasService = this.areasService,
added = this.added;
Make a function that create a promise for your subscribe:
function createAreaPromise(newArea, parentId) {
return new Promise((resolve, reject) => {
areasService.createArea(newArea, parentId)
.subscribe(area => resolve(area));
});
}
Than another function to build multiple an array of promises:
function buildPromises() {
let promises = [];
for (let i = 1; i < (amount + 1); i++) {
newArea.name = name + ' ' + startAt
startAt++
promises.push(createAreaPromise(newArea, parentId));
}
return promises;
}
Then solve them with Promise.all, to obtain the same order from creation
let promises = buildPromises();
Promise.all(promises)
.then(results => {
results.forEach(result => added.emit(result));
});
Here a live example:
function random() {
return Math.floor(Math.random() * 5);
}
function makePromise(index) {
return new Promise((resolve) => {
setTimeout(() => {
resolve(index);
}, random() * 1000);
});
}
function buildPromises() {
let promises = [];
for(let i = 0; i < 5; i++) {
promises.push(makePromise(i));
}
return promises;
}
let promises = buildPromises();
Promise.all(promises)
.then(results => {
results.forEach(result => {
console.log(result);
});
});

Best way to wait for .forEach() to complete

Sometimes I need to wait for a .forEach() method to finish, mostly on 'loader' functions. This is the way I do that:
$q.when(array.forEach(function(item){
//iterate on something
})).then(function(){
//continue with processing
});
I can't help but feel that this isn't the best way to wait for a .forEach() to finish. What is the best way to do this?
If there is no asynchronous code inside the forEach, forEach is not asynchronous, for example in this code:
array.forEach(function(item){
//iterate on something
});
alert("Foreach DONE !");
you will see the alert after forEach finished.
Otherwise (You have something asynchronous inside), you can wrap the forEach loop in a Promise:
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
Credit: #rolando-benjamin-vaz-ferreira
The quickest way to make this work using ES6 would be just to use a for..of loop.
const myAsyncLoopFunction = async (array) => {
const allAsyncResults = []
for (const item of array) {
const asyncResult = await asyncFunction(item)
allAsyncResults.push(asyncResult)
}
return allAsyncResults
}
Or you could loop over all these async requests in parallel using Promise.all() like this:
const myAsyncLoopFunction = async (array) => {
const promises = array.map(asyncFunction)
await Promise.all(promises)
console.log(`All async tasks complete!`)
}
var foo = [1,2,3,4,5,6,7,8,9,10];
If you're actually doing async stuff inside the loop, you can wrap it in a promise ...
var bar = new Promise((resolve, reject) => {
foo.forEach((value, index, array) => {
console.log(value);
if (index === array.length -1) resolve();
});
});
bar.then(() => {
console.log('All done!');
});
If you have an async task inside a loop and you want to wait. you can use for await
for await (const i of images) {
let img = await uploadDoc(i);
};
let x = 10; //this executes after
Use for of instead of forEach. Like this:
for (const item of array) {
//do something
}
console.log("finished");
"finished" will be logged after finishing the loop.
forEach() doesn't return anything, so a better practice would be map() + Promise.all()
var arr = [1, 2, 3, 4, 5, 6]
var doublify = (ele) => {
return new Promise((res, rej) => {
setTimeout(() => {
res(ele * 2)
}, Math.random() ); // Math.random returns a random number from 0~1
})
}
var promises = arr.map(async (ele) => {
// do some operation on ele
// ex: var result = await some_async_function_that_return_a_promise(ele)
// In the below I use doublify() to be such an async function
var result = await doublify(ele)
return new Promise((res, rej) => {res(result)})
})
Promise.all(promises)
.then((results) => {
// do what you want on the results
console.log(results)
})
A universal solution for making sure that all forEach() elements finished execution.
const testArray = [1,2,3,4]
let count = 0
await new Promise( (resolve) => {
testArray.forEach( (num) => {
try {
//some real logic
num = num * 2
} catch (e) {
// error handling
console.log(e)
} fanally {
// most important is here
count += 1
if (count == testArray.length) {
resolve()
}
}
})
})
The idea is same with the answer using index to count. But in real case, if error happened, the index way cannot count correctly. So the solution is more robust.
Thx
const array = [1, 2, 3];
const results = [];
let done = 0;
const asyncFunction = (item, callback) =>
setTimeout(() => callback(item * 10), 100 - item * 10);
new Promise((resolve, reject) => {
array.forEach((item) => {
asyncFunction(item, (result) => {
results.push(result);
done++;
if (done === array.length) resolve();
});
});
}).then(() => {
console.log(results); // [30, 20, 10]
});
// or
// promise = new Promise(...);
// ...
// promise.then(...);
The order of results in the "results" array can be different than the order of items in the original array, depending on the time when the asyncFunction() finishes for each of the items.
Alter and check a counter at the end of every possible unique branch of code, including callbacks. Example:
const fs = require('fs');
/**
* #description Delete files older than 1 day
* #param {String} directory - The directory to purge
* #return {Promise}
*/
async function purgeFiles(directory) {
const maxAge = 24*3600000;
const now = Date.now();
const cutoff = now-maxAge;
let filesPurged = 0;
let filesProcessed = 0;
let purgedSize = 0;
await new Promise( (resolve, reject) => {
fs.readdir(directory, (err, files) => {
if (err) {
return reject(err);
}
if (!files.length) {
return resolve();
}
files.forEach( file => {
const path = `${directory}/${file}`;
fs.stat(path, (err, stats)=> {
if (err) {
console.log(err);
if (++filesProcessed === files.length) resolve();
}
else if (stats.isFile() && stats.birthtimeMs < cutoff) {
const ageSeconds = parseInt((now-stats.birthtimeMs)/1000);
fs.unlink(path, error => {
if (error) {
console.log(`Deleting file failed: ${path} ${error}`);
}
else {
++filesPurged;
purgedSize += stats.size;
console.log(`Deleted file with age ${ageSeconds} seconds: ${path}`);
}
if (++filesProcessed === files.length) resolve();
});
}
else if (++filesProcessed === files.length) resolve();
});
});
});
});
console.log(JSON.stringify({
directory,
filesProcessed,
filesPurged,
purgedSize,
}));
}
// !!DANGER!! Change this line! (intentional syntax error in ,')
const directory = ,'/tmp'; // !!DANGER!! Changeme
purgeFiles(directory).catch(error=>console.log(error));
I'm not sure of the efficiency of this version compared to others, but I used this recently when I had an asynchronous function inside of my forEach(). It does not use promises, mapping, or for-of loops:
// n'th triangular number recursion (aka factorial addition)
function triangularNumber(n) {
if (n <= 1) {
return n
} else {
return n + triangularNumber(n-1)
}
}
// Example function that waits for each forEach() iteraction to complete
function testFunction() {
// Example array with values 0 to USER_INPUT
var USER_INPUT = 100;
var EXAMPLE_ARRAY = Array.apply(null, {length: USER_INPUT}).map(Number.call, Number) // [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, n_final... ] where n_final = USER_INPUT-1
// Actual function used with whatever actual array you have
var arrayLength = EXAMPLE_ARRAY.length
var countMax = triangularNumber(arrayLength);
var counter = 0;
EXAMPLE_ARRAY.forEach(function(entry, index) {
console.log(index+1); // show index for example (which can sometimes return asynchrounous results)
counter += 1;
if (triangularNumber(counter) == countMax) {
// function called after forEach() is complete here
completionFunction();
} else {
// example just to print counting values when max not reached
// else would typically be excluded
console.log("Counter index: "+counter);
console.log("Count value: "+triangularNumber(counter));
console.log("Count max: "+countMax);
}
});
}
testFunction();
function completionFunction() {
console.log("COUNT MAX REACHED");
}
I had to deal with the same problem (forEach using multiple promises inside) and none of the solutions presented at the current date were helpful for me. So I implemented a check array, were each promise updates its complete status. We have a general promise that wraps the process. We only resolve the general promise when each promise completed. Snippet code:
function WaitForEachToResolve(fields){
var checked_fields = new Array(fields.length).fill(0);
const reducer = (accumulator, currentValue) => accumulator + currentValue;
return new Promise((resolve, reject) => {
Object.keys(fields).forEach((key, index, array) => {
SomeAsyncFunc(key)
.then((result) => {
// some result post process
checked_fields[index] = 1;
if (checked_fields.reduce(reducer) === checked_fields.length)
resolve();
})
.catch((err) => {
reject(err);
});
}
)}
}
I like to use async-await instead of .then() syntax so for asynchronous processing of data, modified the answer of #Ronaldo this way -
let finalData = [];
var bar = new Promise(resolve => {
foo.forEach((value, index) => {
const dataToGet = await abcService.getXyzData(value);
finalData[index].someKey = dataToGet.thatOtherKey;
// any other processing here
if (finalData[dataToGet.length - 1].someKey) resolve();
});
});
await Promise.all([bar]);
console.log(`finalData: ${finalData}`);
NOTE: I've modified the if condition where it resolves the promise to meet my conditions. You can do the same in your case.
You can use this, because we are using async/await inside the forEach loop. You can use your own logic inside the loop.
let bar = new Promise((resolve, reject) => {
snapshot.forEach(async (doc) => {
"""Write your own custom logic and can use async/await
"""
const result = await something()
resolve(result);
});
});
let test = []
test.push(bar)
let concepts = await Promise.all(test);
console.log(concepts);
For simple compare code i like use for statement.
doit();
function doit() {
for (var i = 0; i < $('span').length; i++) {
console.log(i,$('span').eq(i).text() );
if ( $('span').eq(i).text() == "Share a link to this question" ) { // span number 59
return;
}
}
alert('never execute');
}
If there are async (observable) method calls inside the for loop, you could use the following method:
await players.reduce(async (a, player) => {
// Wait for the previous item to finish processing
await a;
// Process this item
await givePrizeToPlayer(player);
}, Promise.resolve());
Check here: https://gist.github.com/joeytwiddle/37d2085425c049629b80956d3c618971
I've been using this and it works best .forEach()
//count
var expecting = myArray.length;
myArray.forEach(function(item){
//do logic here
var item = item
//when iteration done
if (--expecting === 0) {
console.log('all done!');
}
})

Categories