Resolve a Promise only after a stream has ended

Resolve a Promise only after a stream has ended - javascript

I'd like to know how I could resolve and go the next array value only after the first stream has ended, my current code just executes streams for all urls instead of doing it one by one
const downloadUrl = (url) => {
console.log(url);
console.log("Started :" + url.name);
const writer = fs.createWriteStream(url.name);
return new Promise(resolve => {
return axios({
url: url.url,
responseType: 'stream',
})
.then(response => {
const totalLength = response.headers['content-length'];
const stream = response.data;
let chunksDone = 0;
stream.on('data', (chunk) => {
const chunkLength = chunk.length;
chunksDone += chunkLength;
const percent = ((chunksDone / totalLength) * 100).toFixed(2);
printProgress(url.name, percent)
writer.write(new Buffer(chunk));
})
return streamToPromise(stream);
})
.then(() => {
debugger;
writer.end();
})
.catch(log);
})
}
const streamToPromise = (stream) => {
return new Promise(function (resolve, reject) {
stream.on("end", resolve);
stream.on("error", reject);
})
}
const urls = ['some.mp3','someother.mp3'];
const promise = urls.reduce(function (acc, item) {
return downloadUrl(item)
}, Promise.resolve);

return new Promise(resolve => {
return axios({
url: url.url,
responseType: 'stream',
})
…
}
This returns a promise that never resolves. Drop the new Promise wrapper and return axios(… directly.
const promise = urls.reduce(function (acc, item) {
return downloadUrl(item)
}, Promise.resolve);
This doesn’t wait for any promises. It looks like you were going for:
const promise = urls.reduce(function (acc, item) {
return acc.then(() => downloadUrl(item));
}, Promise.resolve());
Also, there shouldn’t be a need to copy chunk with new Buffer before writing it.

Related

How to mock https.get with Jest

I have a function that calls https.get inside a promise which I want to test with Jest.
The function is like this:
const request = (url) => {
return new Promise((resolve, reject) => {
const chunks = [];
https.get(url, (stream) => {
stream
.on('data', (chunk) => {
if( chunk ) {
chunks.push(JSON.parse(chunk));
}
})
.on('error', (err) => {
reject(err);
})
.on('end', () => {
const data = doSomething(chunks);
resolve(data)
});
});
})
}
I want to test that when the function resolves on "end" and rejects on "error";
Currently I have a test like this but because .on("end") doesn't get called, the promise never resolves.
describe("request", () => {
it("Should resolve", async (done) => {
const response = await request("my-url");
expect(response).toEqual("some-data")
})
})
How can I mock events like .on("end") to be called and ensure the promise resolves?

You can do something like this.
// ./request.test.js
jest.mock('https', () => ({
methodToMock: {}
}));
const Stream = require('stream');
const request = require("./request");
const httpsMock = require("https");
describe("request", () => {
it("Should resolve", async () => {
var streamStream = new Stream()
httpsMock.get = jest.fn().mockImplementation((url, cb) => {
cb(streamStream)
streamStream.emit('data', 'some');
streamStream.emit('data', '-');
streamStream.emit('data', 'data');
streamStream.emit('end'); // this will trigger the promise resolve
})
const response = await request("my-url");
expect(response).toEqual("some-data");
})
})
const https = require("https");
const request = (url) => {
return new Promise((resolve, reject) => {
const chunks = [];
https.get(url, (stream) => {
stream
.on('data', (chunk) => {
if (chunk) {
// chunks.push(JSON.parse(chunk));
chunks.push(chunk);
}
})
.on('error', (err) => {
reject(err);
})
.on('end', () => {
// const data = doSomething(chunks);
const data = chunks.join('');
resolve(data)
});
});
})
}
module.exports = request;
Note that jest.mock('https', ...) need to be called before const request = require("./request"); if you want https to be mocked.

Async module returning promise [object Promise]

I am trying to export the value with instrument variable. however data is returning as [object Promise] than object. How can I assign module variable with the final result rather than the promise object.
var instruments = {
data: async () => {
return new Promise((resolve, reject) => {
/// Respond after retrieving the data
resolve({result : "...." }
);
}
}
var symbols = async () => {
const res = await instruments.data();
return res;
}
module.exports.instrument = symbols().then((data) => {
console.log('data');
return data;
}).catch((e) => {
console.log('error');
return {}
});

It looks like you want a singleton cache. Here is a basic implementation
cache.js
let data = {}
module.exports = {
getData: () => {
return data
},
setData: newData => {
data = newData
return
},
}
No need for async here. I would separate this code with the code that retrieves data.
fetchData.js
const cache = require('./cache')
const fetchData = () => {} // fetch data code here
fetchData().then(data => {
cache.setData(data)
})

try this
var instruments = {
data: () => {
return new Promise((resolve, reject) => {
/// Respond after retrieving the data
resolve({result : "...." });
}
}
var symbols = async () => {
const res = await instruments.data();
return res;
}
module.exports.instrument = symbols;
then import instrument method to call and then call
const instrument = require("./filepath");
instrument().then((data) => {
console.log('data');
}).catch((e) => {
console.log(e);
});

If your async function instruments.data() called, it'll await return Promise.
just append await at return for your expected result.
var instruments = {
data: async () => {
return await new Promise((resolve, reject) => {
// Respond after retrieving the data
resolve({result : "...." });
}
}
or remove async. it's same as above.
var instruments = {
data: () => {
return new Promise((resolve, reject) => {
// Respond after retrieving the data
resolve({result : "...." });
}
}

How to excecute asyncronous promise used Promise.All()?

I want to ask, how do you run ascyronus promises in the promise all? when I run the function that I made Promise.all runs all the functions in parallel (syncronus)
if you run the code below then the result is
// jon
// andrey
// tania
// JON
// ANDREY
// TANIA
here is my code
// First promise returns an array
const getUsers = () => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve([{ id: 'jon' }, { id: 'andrey' }, { id: 'tania' }]), 600)
})
}
// Second promise relies on the resulting array of first promise
const getIdFromUser = users => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(users.id), 500)
})
}
// Third promise relies on the result of the second promise
const capitalizeIds = id => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(id.toUpperCase()), 200)
})
}
const newPromise = async user => {
const userId = await getIdFromUser(user)
console.log(userId) //jon
const capitalizedId = await capitalizeIds(userId)
console.log(capitalizedId) //JON
return
}
const runAsyncFunctions = async () => {
const users = await getUsers()
await Promise.all(users.map(user => newPromise(user)))
}
runAsyncFunctions()
but I want the results as below, this will work if the promise all is run asyncronus
// jon
// JON
// andrey
// ANDREY
// tania
// TANIA
and this is my playground code playground

If you want to initialize a Promise only after the last Promise has finished - that is, run them in serial, not in parallel - then .map and Promise.all is not the right tool for the job, because that'll initialize all Promises at once. await in a for loop instead:
const runAsyncFunctions = async() => {
const users = await getUsers()
for (const user of users) {
await newPromise(user);
}
}
// First promise returns an array
const getUsers = () => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve([{
id: 'jon'
}, {
id: 'andrey'
}, {
id: 'tania'
}]), 600)
})
}
// Second promise relies on the resulting array of first promise
const getIdFromUser = users => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(users.id), 500)
})
}
// Third promise relies on the result of the second promise
const capitalizeIds = id => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(id.toUpperCase()), 200)
})
}
const newPromise = async user => {
const userId = await getIdFromUser(user)
console.log(userId) //jon
const capitalizedId = await capitalizeIds(userId)
console.log(capitalizedId) //JON
return
}
const runAsyncFunctions = async() => {
const users = await getUsers()
for (const user of users) {
await newPromise(user);
}
}
runAsyncFunctions()
If your real code allows you to do the logging outside, you can keep running the Promises in parallel, but only log the results in order once they're all completed:
const newPromise = async user => {
const userId = await getIdFromUser(user)
const capitalizedId = await capitalizeIds(userId)
return [userId, capitalizedId];
}
const runAsyncFunctions = async() => {
const users = await getUsers()
const names = await Promise.all(users.map(user => newPromise(user)));
for (const name of names.flat()) {
console.log(name);
}
}
// First promise returns an array
const getUsers = () => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve([{
id: 'jon'
}, {
id: 'andrey'
}, {
id: 'tania'
}]), 600)
})
}
// Second promise relies on the resulting array of first promise
const getIdFromUser = users => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(users.id), 500)
})
}
// Third promise relies on the result of the second promise
const capitalizeIds = id => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(id.toUpperCase()), 200)
})
}
const newPromise = async user => {
const userId = await getIdFromUser(user)
const capitalizedId = await capitalizeIds(userId)
return [userId, capitalizedId];
}
const runAsyncFunctions = async() => {
const users = await getUsers()
const names = await Promise.all(users.map(user => newPromise(user)));
for (const name of names.flat()) {
console.log(name);
}
}
runAsyncFunctions()

Promise all runs like a funny wrapper around a normal array interator function...
let results = Promise.all(myarray.map(item => {
try {
return await functionThatReturnsAPromise()
} catch (error) {
console.log(error)
return null
}
}))
A good alternative to Promise all is for of because you can put await on it...
for await (let item of myarray) {
// Try catch etc
}
Node will be fine with this, but you might want to check browser support if you're running it in the front-end.

the getIdFromUser is executed async. This means on the next line, the whole array is already there and that is why it is printed in a serial manner. Fetching both the userId and the capitalized firstly will result in what you want
// First promise returns an array
const getUsers = () => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve([{ id: 'jon' }, { id: 'andrey' }, { id: 'tania' }]), 600)
})
}
// Second promise relies on the resulting array of first promise
const getIdFromUser = async users => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(users.id), 0)
})
}
// Third promise relies on the result of the second promise
const capitalizeIds = id => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(id.toUpperCase()), 0)
})
}
const newPromise = async user => {
const userId = await getIdFromUser(user)
const capitalizedId = await capitalizeIds(userId)
console.log(userId) //jon
console.log(capitalizedId) //JON
return
}
const runAsyncFunctions = async () => {
const users = await getUsers()
console.log(users)
await Promise.all(users.map(user => newPromise(user)))
}
runAsyncFunctions()

I have tried to make it simple and removed functions that are not required, I can prodcue the requested output in question
You can alos check here solution
// First promise returns an array
const getUsers = () => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve([{ id: 'jon' }, { id: 'andrey' }, { id: 'tania' }]), 600)
})
}
// Second promise relies on the resulting array of first promise
const getIdFromUser = users => {
return new Promise((resolve, reject) => {
return setTimeout(() => resolve(users.id), 500)
})
}
// You do not need this, why make it complicated ??
// Third promise relies on the result of the second promise
// const capitalizeIds = async id => {
// return new Promise((resolve, reject) => {
// return setTimeout(() => resolve(id.toUpperCase()), 200)
// })
// }
function capitalizeIds(id)
{
return id.toUpperCase();
}
const runAsyncFunctions = async () => {
const users = await getUsers()
users.map(user => {
console.log(user.id) //jon
const capitalizedId = capitalizeIds(user.id);
console.log(capitalizedId) //JON
})
}
runAsyncFunctions()

Converting Promises Code to async await and mocking test cases?

How to convert the below function which has nested promise and await to just using await or only using promises ?
const test = (url, id) => {
return new Promise((_resolve, _reject) => {
if (isValidUrl(url)) {
let storage = new Storage(Indexdb, id);
const cae = new valueExtract(url);
cae.fetch()
.then(data => {
new zip(data)
.then(obj => obj.getZip())
.then(obj => obj.getList())
.then(list => {
return new Promise(async (resolve, reject) => {
try {
let sI = storage.connection;
await Promise.all(Object.keys(list).map(async (fileName, index) => {
let blob = await new FileExtractor(list[fileName]);
await sI.setItemForce(
fileName,
new StoreObject(
fileName,
'testData',
blob
).dataObject
)
}))
_resolve(sI);
} catch (err) {
_reject(err)
}
})
})
.catch(err => _reject(err))
})
.catch(err => _reject(err))
} else {
_reject('Invalid URL')
}
})
};
I was unable to do the same this what i tried but it never resolves
const test = async (url, id) => {
if (isValidUrl(url)) {
try {
let storage = new Storage(Indexdb, id);
const cae = new valueExtract(url);
const data = await cae.fetch();
return new ZIPExtractor(data)
.then(obj => obj.getZip())
.then(obj => obj.getList())
.then(list => {
return async (resolve, reject) => {
try {
let sI = storage.connection;
await Promise.all(Object.keys(list).map(async (fileName, index) => {
let blob = await new FileExtractor(list[fileName]);
await sI.setItemForce(
fileName,
new StoreObject(
fileName,
'testData',
blob
).dataObject
)
}))
} catch (err) {
throw new Error(err)
}
}
})
.catch(err => _reject(err))
} catch (e) {
throw new Error('Invalid URL')
}
}
};
Also how do we write test case for these kind of a function so that we need not pass in actual network url and mock in jest.

It should fulfill, but with the async (resolve, reject) { … } that you return. You never should've used this in the first place, you can just omit it:
const test = async (url, id) => {
if (!isValidUrl(url)) {
throw new Error('Invalid URL')
}
const storage = new Storage(Indexdb, id);
const cae = new valueExtract(url);
const data = await cae.fetch();
const obj = await new ZIPExtractor(data); // shudder. A constructor should never return a promise
const zip = await obj.getZip();
const list = await zip.getList();
const sI = storage.connection;
await Promise.all(Object.keys(list).map(async (fileName, index) => {
const blob = await new FileExtractor(list[fileName]);
const store = new StoreObject(fileName, 'testData', blob);
await sI.setItemForce(fileName, store.dataObject);
}));
return sI; // or something?
}

ES6 Promise wait for K out N promises to resolve

var p1 = new Promise((resolve, reject) => {
setTimeout(resolve, 1000, 'one');
});
var p2 = new Promise((resolve, reject) => {
setTimeout(resolve, 2000, 'two');
});
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 3000, 'three');
});
Promise.all([p1,p2,p3]).then(values => {
console.log(values);
}, reason => {
console.log(reason)
});
How can i wait for 2 promises to get completed ? Promise.race() wait for one promise to get completed.
Edit
I have n number of promises, what i want to achieve is wait for first k number of promises to get resolved and than trigger some event. assume k < n
Edit - 2
I am sure that k number of promise will be successfully resolved out of n numbers of promise given

(Note: Bluebird has a built-in helper function that serves precisely this purpose and has more or less the same behavior as my waitForN method below, so that option is always available)
I don't believe ES6 promises have an elegant built-in way to do this, but you could define the following relatively short now somewhat long helper function to take care of this.
Edit: Added an additional helper function that includes the indices of the successful promises in the result.
Edit: Updated to reject if the number of rejections reaches the point where successful resolution would be impossible.
Edit: Updated to more gracefully work with promises if it is iterable and check edge cases where the result should immediately resolve (e.g. if n === 0)
function waitForN(n, promises) {
let resolved = [];
let failCount = 0;
let pCount = 0;
return new Promise((resolve, reject) => {
const checkSuccess = () => {
if (resolved.length >= n) { resolve(resolved); }
};
const checkFailure = () => {
if (failCount + n > pCount) {
reject(new Error(`Impossible to resolve successfully. n = ${n}, promise count = ${pCount}, failure count = ${failCount}`));
}
};
for (let p of promises) {
pCount += 1;
Promise.resolve(p).then(r => {
if (resolved.length < n) { resolved.push(r); }
checkSuccess();
}, () => {
failCount += 1;
checkFailure();
});
}
checkFailure();
checkSuccess();
});
}
const waitForNWithIndices = (n, promises) =>
waitForN(n, promises.map((p, index) =>
Promise.resolve(p).then(result => ({ index, result }))
));
var p1 = new Promise((resolve, reject) => {
setTimeout(resolve, 1000, 'one');
});
var p2 = new Promise((resolve, reject) => {
setTimeout(resolve, 2000, 'two');
});
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 3000, 'three');
});
var p4 = new Promise((resolve, reject) => {
setTimeout(resolve, 1500, 'four');
});
waitForN(2, [p1,p2,p3,p4]).then(values => {
console.log(values);
}, reason => {
console.log(reason)
});
waitForNWithIndices(2, [p1,p2,p3,p4]).then(values => {
console.log(values);
}, reason => {
console.log(reason)
});

You could use a higher order function to bake in the n then run all promises waiting for n to finish
var p1 = new Promise((resolve, reject) => {
setTimeout(resolve, 1000, 'one');
});
var p2 = new Promise((resolve, reject) => {
setTimeout(resolve, 2000, 'two');
});
var p3 = new Promise((resolve, reject) => {
setTimeout(resolve, 3000, 'three');
});
const raceN = n => (...promises) => {
const resolved = []
return new Promise((res, rej) =>
promises.forEach(promise =>
promise.then(x => {
resolved.push(x)
if (resolved.length === n)
res(resolved)
})
.catch(rej)
)
)
}
const race2 = raceN(2)
race2(p1, p2, p3)
.then(results => console.log(results))
race2(Promise.resolve('resolved'), Promise.reject('rejected'))
.then(results => console.log(results))
.catch(reason => console.log(reason))

Given my many comments on JLRishe's answer, I'll also post my subtly different version of the function:
function any(k, iterable) {
const results = [];
let fullfilled = 0, rejected = 0, n = 0;
return new Promise((resolve, reject) => {
k = Math.max(0, Math.floor(k));
function check() {
if (fulfilled == k)
resolve(results);
else if (rejected + k == n + 1)
reject(new Error(`No ${k} of ${n} got fulfilled`));
}
for (const thenable of iterable) {
const i = n++;
Promise.resolve(thenable).then(res => {
if (fulfilled++ < k) results[i] = res;
check();
}, () => {
rejected++;
check();
});
}
check();
});
}

A bit late but maybe this will do it for you:
const first = howMany => (promises,resolved=[],rejected=[]) => {
if(promises.length-rejected.length<howMany){
return Promise.reject([resolved,rejected]);
}
if(resolved.length===howMany){
return Promise.resolve(resolved);
}
if(resolved.length===0&&rejected.length===0){
promises=promises.map(
(p,index)=>Promise.resolve(p)
.then(resolve=>[resolve,index])
.catch(err=>Promise.reject([err,index]))
);
}
return Promise.race(promises)
.then(
([resolve,index])=>
first(howMany)(
promises.filter(
(p,i)=>i!==index
),
resolved.concat([resolve]),
rejected
)
)
.catch(
([err,index])=>
first(howMany)(
promises.filter(
(p,i)=>i!==index
),
resolved,
rejected.concat([err])
)
);
};
const first2 = first(2);
first2([p1,p2,p3])
.then(
result=>//you will have an array of 2 with resolve value(s)
)
.catch(
err=>//you will have 2 arrays, resolved ones and errors
)

EDIT: Updated to match question edits.
Pretty easy to roll your own when the abstraction limits your expressiveness.
This code is short and can easily be reworked to run in legacy ES3 environments.
You can also easily expand its usefulness by adding code to cancel unresolved timeouts. This is what you get when you don't hand everything over to a library; greater simplicity and more capability.
function raceN(n, fns, resolve, reject) {
const res = [], rej = [];
let halt = false;
for (const [idx, fn] of fns.entries()) {
fn(data => update(res, data, idx), data => update(rej, data, idx));
}
function update(arr, data, idx) {
if (halt) return;
arr.push({idx, data});
if ((halt=res.length >= n)) resolve(res);
else if ((halt=rej.length > fns.length - n)) reject(rej);
}
}
DEMO:
function raceN(n, fns, resolve, reject) {
const res = [], rej = [];
let halt = false;
for (const [idx, fn] of fns.entries()) {
fn(data => update(res, data, idx), data => update(rej, data, idx));
}
function update(arr, data, idx) {
if (halt) return;
arr.push({idx, data});
if ((halt=res.length >= n)) resolve(res);
else if ((halt=rej.length > fns.length - n)) reject(rej);
}
}
function rand() { return Math.ceil(Math.random() * 5000) }
var fns = [(resolve, reject) => {
setTimeout(resolve, rand(), 'one');
},
(resolve, reject) => {
setTimeout(resolve, rand(), 'two');
},
(resolve, reject) => {
setTimeout(reject, rand(), 'three');
}];
raceN(2, fns, values => {
console.log("SUCCESS:", values);
}, reason => {
console.log("REJECT:", reason)
});
If you're guaranteed that at least n elements will succeed, it then becomes even shorter and simpler.
function raceN(n, fns, resolve, reject) {
const res = [];
for (const [idx, fn] of fns.entries()) {
fn(data => res.length < n && res.push({idx, data}) == n) && resolve(res),
data => reject({idx, data}));
}
}
DEMO:
function raceN(n, fns, resolve, reject) {
const res = [];
for (const [idx, fn] of fns.entries()) {
fn(data => res.length < n && res.push({idx, data}) == n) && resolve(res),
data => reject({idx, data}));
}
}
function rand() { return Math.ceil(Math.random() * 4000) }
var fns = [(resolve, reject) => {
setTimeout(resolve, rand(), 'one');
},
(resolve, reject) => {
setTimeout(resolve, rand(), 'two');
},
(resolve, reject) => {
setTimeout(reject, rand(), 'three');
}];
raceN(2, fns, values => {
console.log("SUCCESS:", values);
}, reason => {
console.log("REJECTING:", reason)
});

One other idea could be using Promise.reject() to cut it short when the condition is met while catching possible errors thrown, could be as follows;
var p1 = new Promise((v, x) => setTimeout(v, 1000, 'one')),
p2 = new Promise((v, x) => setTimeout(v, 5000, 'two')),
p3 = new Promise((v, x) => setTimeout(v, 1500, 'three')),
pn = (n, ps, k = 0, r = []) => Promise.all(ps.map(p => p.then(v => (k === n - 1 ? Promise.reject(r.concat(v))
: (++k, r.push(v))))))
.catch(r => Array.isArray(r) ? r : Promise.reject(r));
pn(2,[p1,p2,p3]).then(rs => console.log(rs))
.catch(e => console.log(e));

A more modern approach, in 2022, using iter-ops library:
import {pipeAsync, take, waitRace} from 'iter-ops';
// easy-to-read way to define a job function:
const job = (delay, value) => new Promise(resolve => {
setTimeout(() => resolve(value), delay);
});
// list of jobs:
const input = [
job(1000, 'one'),
job(2000, 'two'),
job(3000, 'three')
];
const i = pipeAsync(
input, // our input
waitRace(10), // race-resolve with cache up to 10 items
take(2) // take only the first 2 items
).catch(err => {
console.log(err); // report any iteration-time error
});
// iterating through our list of jobs:
(async function () {
for await(const a of i) {
console.log(a);
}
})();
This will output one and two. You can twist the delays to see the result change accordingly.
CREDITS
I am the author of the library, while operator waitRace was implemented by #Bergi.

We Keep Coding

JavaScript is the programming language of the Web.

Resolve a Promise only after a stream has ended - javascript

Related

How to mock https.get with Jest

Async module returning promise [object Promise]

How to excecute asyncronous promise used Promise.All()?

Converting Promises Code to async await and mocking test cases?

ES6 Promise wait for K out N promises to resolve

Categories

Resources