I'm trying to use the Fetch API readable streams to download multiple files.
const files = [.....]; // my file objects
const promises = [];
for (let i = 0; i < files.length; i += 1) {
const file = files[i];
promises.push(
fetch(file.uri)
.then(response => {
const reader = response.body.getReader();
return new ReadableStream({
async start(controller) {
while (true) {
const { done, value } = await reader.read();
// When no more data needs to be consumed, break the reading
if (done) {
break;
}
if (!file.content) {
file.content = value;
}
else {
file.content += value;
}
// Enqueue the next data chunk into our target stream
controller.enqueue(value);
}
// Close the stream
controller.close();
reader.releaseLock();
}
});
})
.then(rs => new Response(rs))
);
}
return Promise.all(promises).then(() => {
// do something else once all the files are downloaded
console.log('All file content downloaded');
});
The idea here is for the file objects to only have a URI. Then this code will add a content field. After this point I can do something else.
However, in practice, the code sequence is incorrect. Namely the following lines run immediately after each other.
return new ReadableStream({...});
console.log('All file content downloaded');
The code's not waiting until the file content has been downloaded. The log above is printed ahead of time. After it runs I see the code hit the
while (true) {
Loop's code. i.e. Where the file content is streamed.
I'm obviously misunderstanding a fundamental concept. How can I wait for the file content to be downloaded and then do something else? i.e. How do streams work with the promise.then() model.
Found the simplest solution was to create my own promise.
const files = [.....]; // my file objects
const promises = [];
for (let i = 0; i < files.length; i += 1) {
const file = files[i];
promises.push(
fetch(file.uri)
.then(response => {
return new Promise((resolve, reject) => {
const reader = response.body.getReader();
const stream = new ReadableStream({
async start(controller) {
while (true) {
const { done, value } = await reader.read();
// When no more data needs to be consumed, break the reading
if (done) {
break;
}
if (!file.content) {
file.content = value;
}
else {
file.content += value;
}
// Enqueue the next data chunk into our target stream
controller.enqueue(value);
}
// Close the stream
controller.close();
reader.releaseLock();
}
});
});
})
.then(rs => new Response(rs))
);
}
return Promise.all(promises).then(() => {
// do something else once all the files are downloaded
console.log('All file content downloaded');
});
Caveat: The error scenario should also be gracefully handled.
Related
I am trying to do the following actions in order:
1. Create filename
2. Copy that filename to the clipboard
3. Open a window to a specific url
4. Click a specific div on the new window
5. Close the window
6. Now, I can print and simply paste the copied filename and finally print to pdf
I need these actions to happen one after the other. I need to run this logic inside of a loop (or at least execute it 200 times). So I need these steps to happen and THEN move to the next item in the html table that I am traversing.
let obj = {};
const copyToClipboard = (text) => {
const tempElem = document.createElement('input');
document.body.appendChild(tempElem);
tempElem.setAttribute('value', text);
tempElem.select();
document.execCommand("copy");
document.body.removeChild(tempElem);
};
const traverse = (i, inObj) => {
const number = document.getElementById('nid').rows[i].getElementsByTagName('td')[1].innerText;
const status = document.getElementById('nid').rows[i].getElementsByTagName('td')[4].innerText;
const file = `Item: ${name}`;
if(inObj[number] == "undefined" || inObj[poName] != status){
inObj[number] = status;
copyToClipboard(file);
let url = 'mysite.com/myroute';
let a = window.open(url);
a.focus();
let timer = setInterval(() => {
a.document.getElementsByClassName('anotherid')[1].click();
const i = a.document.getElementById('myframe');
i.contentWindow.focus();
i.contentWindow.print();
a.close();
clearInterval(timer);
console.log('Success!');
}, 1000);
} else{
console.log('Failure!');
}
};
for(let i = 0; i < tableSize; i++{
traverse(i, obj);
}
Some pieces of my code will execute before the others. For example, the windows will all open at once and then the remaining actions will take place. I need this code to execute completely inside of a loop before the next index iteration.
The only asynchronous thing I see in your code is traverse's completion. So simply define traverse to return a promise. Given the code in traverse, this is probably easiest if you do it explicitly:
const traverse = (i, inObj) => {
return new Promise((resolve, reject) => { // <===========================
const number = document.getElementById('nid').rows[i].getElementsByTagName('td')[1].innerText;
const status = document.getElementById('nid').rows[i].getElementsByTagName('td')[4].innerText;
const file = `Item: ${name}`;
if(inObj[number] == "undefined" || inObj[poName] != status){
inObj[number] = status;
copyToClipboard(file);
let url = 'mysite.com/myroute';
let a = window.open(url);
a.focus();
let timer = setInterval(() => {
a.document.getElementsByClassName('anotherid')[1].click();
const i = a.document.getElementById('myframe');
i.contentWindow.focus();
i.contentWindow.print();
a.close();
clearInterval(timer);
console.log('Success!');
resolve(); // <===========================
}, 1000);
// *** Probably want to do a timeout here in case the window never loads
} else{
console.log('Failure!');
reject(); // <===========================
}
});
};
Then:
If you can use async syntax, write your loop using await on the promise returned by traverse. Example:
// In an `async` function
try {
for (let i = 0; i < tableSize; ++i) {
await traverse(i, obj);
}
// Done
} catch (error) {
// An error occurred
}
If not, chain your operations together by calling then on the promise returned by traverse:
let promise = Promise.resolve();
for (let i = 0; i < tableSize; ++i) {
promise = promise.then(() => traverse(i, obj));
}
promise
.then(() => {
// Done
})
.catch(error => {
// An error occurred
};
Updating `tra
there is a function that I use to read all files in a directory and then sent an object with emitter to the client.
this is my code that works fine,
const getFilesList = (path, emitter) => {
fs.readdir(path, (err, files) => {
emitter('getFileList', files);
});
};
but when I want to filter hidden files with this code, the 'standardFolders' will send empty in the emitter.
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
});
});
} else {
standardFolders = null;
}
emitter('getFileList', standardFolders);
});
};
what is wrong with my code in the second part?
winattr.get(filepath,callback) is asynchronous, so imagine your code "starts" the file.map() line and then immediately skips to emitter('getFileList',standardFolders) --- which standardFolders is empty because it hasn't finished yet!
You can use a library like async.io to handle your callback functions, or you can use a counter and keep track of when all of the callbacks (for each file) has finished yourself.
Example:
// an asynchronous function because setTimeout
function processor(v,cb){
let delay = Math.random()*2000+500;
console.log('delay',delay);
setTimeout(function(){
console.log('val',v);
cb(null,v);
},delay);
}
const main = function(){
const list = ['a','b','c','d'];
let processed = [];
let count = 0;
console.log('starting');
list.map(function(v,i,a){
console.log('calling processor');
processor(v,function(err,value){
processed.push(v);
count+=1;
console.log('count',count);
if(count>=list.length){
// all are finished, continue on here.
console.log('done');
}
})
})
console.log('not done yet!');
};
main();
Similarly, for your code:
const getFilesList = (path, emitter) => {
let standardFolders = [];
fs.readdir(path, (err, files) => {
if (files) {
let count = 0;
files.map((file) => {
winattr.get(path + file, function (err, attrs) {
if (err == null && attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
count+=1;
if(count>=files.length){
// finally done
emitter('getFileList', standardFolders);
}
});
});
} else {
standardFolders = null;
emitter('getFileList', standardFolders);
}
});
};
As already sayed in the other answer winattr.get is async, so the loop finishes before any of the callbacks of winattr.get is called.
You could convert your code using async/await and primitify into a code that looks almost like a sync version, and you can completely get rid of the callbacks or counters
const {promisify} = require('util')
const readdir = promisify(require('fs').readdir)
const winattrget = promisify(require('winattr').get)
const getFilesList = async (path, emitter) => {
let standardFolders = [];
try {
let files = await readdir(path);
for (let file of files) {
try {
let attrs = await winattrget(path + file)
if (attrs.directory && (!attrs.hidden && !attrs.system)) {
standardFolders.push(file)
}
} catch (err) {
// do nothing if an error occurs
}
}
} catch (err) {
standardFolders = null;
}
emitter('getFileList', standardFolders);
};
An additional note: In your code you write files.map, but mapping is use to transform the values of a given array and store them in a new one, and this is not done in your current code, so in the given case you should use a forEach loop instead of map.
I need to create a function that runs a 'getFile' function on each item in an array. The getFile function logs 'File contents of x' x being whatever element is in the array.
Currently, I have a working function that runs the getFile on the array and waits for the final response before logging the results.
However, I now need to log the responses as I receive them in order. For example, if my array is [1, 2, 3, 4, 5] currently it logs 'File contents of x' in a random order, so if it was to return the logs, 3 then 4 then 1. As soon as I receive 1, I need to log that, then once I receive 2 logs that and so on.
I will insert my current code below. The problem I'm having is I need to know when the 'empty space' in my array becomes populated so I can log it in real time. Therefore allowing my user to see the result build up rather than just having to wait until all the responses have come back
function fetchContentOfFiles(fileNames, testCB) {
const fileContent = [];
let counter = 0;
fileNames.forEach((file, i) => {
getFile(file, (err, fileName) => {
if (err) console.log(err)
else {
fileContent[i] = fileName;
counter++
if (counter === fileNames.length) {
testCB(null, fileContent)
};
console.log(fileContent)
};
});
});
};
The cleanest way to write this would be to use a for loop inside an async function. Promisify getFile so that it returns a Promise, then await it in every iteration of the loop. At the end of the loop, call the callback:
const getFileProm = file => new Promise((resolve, reject) => {
getFile(file, (err, fileName) => {
if (err) reject(err);
else resolve(fileName);
});
});
async function fetchContentOfFiles(fileNames, testCB) {
const fileContent = [];
try {
for (let i = 0; i < fileNames.length; i++) {
fileContent.push(
await getFileProm(fileNames[i])
);
}
} catch(e) {
// handle errors, if you want, maybe call testCB with an error and return?
}
testCB(null, fileContent);
}
It would probably be even better if fetchContentOfFiles was called and handled as a Promise rather than with callbacks, and then the errors can be handled in the consumer:
async function fetchContentOfFiles(fileNames) {
const fileContent = [];
for (let i = 0; i < fileNames.length; i++) {
fileContent.push(
await getFileProm(fileNames[i])
);
}
return fileContent;
}
fetchContentOfFiles(arr)
.then((fileContent) => {
// do stuff with fileContent
})
.catch((err) => {
// something went wrong
});
I want to return the files of a directory.
I need it to pass the route to another function.
In other words, how can I return the files of a directory using JavaScript/Node.js?
const fs = require('fs');
const path = require('path');
const mdLinks = require('../index');
exports.extension = (route) => {
return new Promise((resolve, reject) => {
try {
recursive(route);
} catch (e) {
reject(e);
}
});
}
const recursive = (route) => {
const extMd = ".md";
let extName = path.extname(route);
let files = [];
fs.stat(route, (err, stats) => {
if (stats && stats.isDirectory()) {
fs.readdir(route, (err, files) => {
files.forEach(file => {
let reFile = path.join(route, file);
if (file !== '.git') {
recursive(reFile);
}
});
})
}
else if (stats.isFile() && (extMd == extName)) {
files.push(route);
}
})
return files;
}
There are multiple problems.
First off, your function is asynchronous so it cannot just return the files value because your function returns long before anything is added to the files array (that's one reason that it's empty). It will have to either call a callback when its done or return a promise that the caller can use. You will have to fix that in both the top level and when you call yourself recursively.
Second, you are redefining files at each recursive step so you have no way to collect them all. You can either pass in the array to add to or you can define the files array at a higher level where everyone refers to the same one or you can have the call to recursive concat the files that are returned to your current array.
Third, you haven't implemented any error handling on any of your asynchronous file I/O calls.
Here's my recommended way of solving all these issue:
exports.extension = (route) => {
return recursive(route);
}
const util = require('util');
const stat = util.promisify(fs.stat);
const readdir = util.promisify(fs.readdir);
// returns a promise that resolves to an array of files
async function recursive(route) {
const extMd = ".md";
let extName = path.extname(route);
let files = [];
let stats = await stat(route);
if (stats.isDirectory()) {
let dirList = await readdir(route);
for (const file of dirList) {
if (file !== '.git') {
let reFile = path.join(route, file);
let newFiles = await recursive(reFile);
// add files onto the end of our list
files.push(...newFiles);
}
}
} else if (stats.isFile() && extMd === extName) {
files.push(route);
}
// make the files array be the resolved value
return files;
});
I created a node application which does is that it scraps google and downloads top 15 images and then store it in HDD in a folder which is the query received after compressing. Now problem that I'm facing is When I'm going back to read that folder using readdirSync and storing the results in error, it returns an empty array, what is wrong with the code.
request(url, function (error, response, body) {
if (!error) {
var $ = cheerio.load(body);
var imgNodes = $('#ires td a img');
// imgNodes is merely an array-like object, sigh.
// This is purposedly old-school JS because newer stuff doesn't work:
var urls = [];
for(let i = 0; i <= 14; i++){
let imgNode = imgNodes[i];
urls.push(imgNode.attribs['src']);
}
// console.log(urls);
const processCompress = new Promise(resolve => {
fs.mkdir(path.join(__dirname,'Photos',query), function (error) {
let j = 0;
if(!error){
for(i in urls){
console.log(i);
var source = tinify.fromUrl(urls[i]);
source.toFile(path.join(__dirname,'Photos', query,"optimized_"+ ++j +".jpg"));
}
}});
resolve();
});
const getFiles = new Promise(resolve => {
fs.readdirSync(path.join(__dirname,'Photos', query)).forEach(function (file) {
fileName.push(path.join(__dirname,'Photos',query,file));
});
resolve();
});
function colourMeBw(){
for(let k = 0; k < fileName.length; k++){
Jimp.read(fileName[k], (err, image) => {
if (err) throw err;
image.greyscale().write(fileName[k]);
});
}}
processCompress.then(() => getFiles);
colourMeBw();
} else {
console.log("We’ve encountered an error: " + error);
}
There are a number of things wrong with your code:
In processCompress(), you are resolving the promise before fs.mkdir() is done and before any of the images have been fetched and written.
In getFiles() you are wrapping a synchronous I/O function in a promise. The first problem is that you shouldn't be using synchronous I/O at all. That is the fastest way to wreck the scalability of your server. Then, once you switch to the async version of fs.readdir(), you have to resolve the promise appropriately.
There's no way to know when colourMeBw() is actually done.
You should never iterate an array with for(i in urls) for a variety of reasons. In ES6, you can use for (url of urls). In ES5, you can use either a traditional for (var i = 0; i < urls.length; i++) {} or urls.forEach().
You have no error propagation. The whole process would choke if you got an error in the middle somewhere because later parts of the process would still continue to try to do their work even though things have already failed. There's no way for the caller to know what errors happened.
There's no way to know when everything is done.
Here's a version of your code that uses promises to properly sequence things, propagate all errors appropriately and tell you when everything is done. I don't myself know the tinify and Jimp libraries so I consulted their documentation to see how to use them with promises (both appear to have promise support built-in). I used the Bluebird promise library to give me promise support for the fs library and to take advantage of Promise.map() which is convenient here.
If you didn't want to use the Bluebird promise library, you could promisify the fs module other ways or event promisify individual fs methods you want to use with promises. But once you get used to doing async programming with promises, you're going to want to use it for all your fs work.
This is obviously untested (no way to run this here), but hopefully you get the general idea for what we're trying to do.
const Promise = require('bluebird');
const fs = Promise.promisifyAll(require('fs'));
const Jimp = require('jimp');
const rp = require('request-promise');
rp(url).then(function(body) {
var $ = cheerio.load(body);
var imgNodes = $('#ires td a img');
// imgNodes is merely an array-like object, sigh.
// This is purposedly old-school JS because newer stuff doesn't work:
var urls = [];
for (let i = 0; i <= 14; i++) {
let imgNode = imgNodes[i];
urls.push(imgNode.attribs['src']);
}
// console.log(urls);
const processCompress = function() {
return fs.mkdirAsync(path.join(__dirname, 'Photos', query).then(function(error) {
let j = 0;
return Promise.map(urls, function(url) {
var source = tinify.fromUrl(url);
return source.toFile(path.join(__dirname, 'Photos', query, "optimized_" + ++j + ".jpg"));
});
});
});
const getFiles = function() {
return fs.readdirAsync(path.join(__dirname, 'Photos', query).then(function(files) {
return files.map(function(file) {
return path.join(__dirname, 'Photos', query, file);
});
});
};
function colourMeBw(fileList) {
return Promise.map(fileList, function(file) {
return Jimp.read(file).greyscale().write(file);
});
}
return processCompress().then(getFiles).then(colourMeBw);
}).then(function() {
// all done here
}).catch(function(err) {
// error here
});
Your query variable in use here does not appear to be defined anywhere so I am assuming it is defined in a higher scope.
Note that one big advantage of using promises for a multi-stage operation like this is that all errors end up on one single place, no matter where they occurred in the overall multi-level process.
Note: If you are processing a large number of images or a medium number of large images, this could end up using a fair amount of memory because this code processes all the images in parallel. One of the advantages of Bluebird's Promise.map() is that is has an optional concurrency option that specifies how many of the requests should be "in-flight" at once. You can dial that down to a medium number to control memory usage if necessary.
Or, you could change the structure so that rather than compress all, then convert all to greyscale, you could compress one, convert it grey scale, then move on to the next, etc...
I read the code and I think what you are trying to do is something like this:
const cheerio = require("cheerio");
const fetch = require("node-fetch");
const tinify = require("tinify");
const fs = require("fs");
const path = require("path");
const getImages = url => {
return fetch(url)
.then(responseToText)
.then(bodyToImageUrls)
.then(makePhotoDirectory)
.then(downloadFiles)
.then(processImageData)
.then(doImageManipulation)
.catch(error => {
console.log("We’ve encountered an error: " + error);
});
};
const responseToText = res => res.text();
const bodyToImageUrls = body => {
const $ = cheerio.load(body);
return $("img").attr("src");
};
const imgNodesToUrls = imgNodes => {
return imgNodes.map(imgNode => imgNode.name);
};
const makePhotoDirectory = urls => {
const dir = path.join(__dirname, "Photos");
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}
return urls;
};
const downloadFiles = urls => {
/*
I could not run this since I don't have a Tinify API key
but I assume that it returns a promise.
*/
const promises = urls.map(url => tinify.fromUrl(url));
return Promise.all(promises);
};
const processImageData = imageData => {
const promises = imageData.map((data, i) => {
const fileUrl = path.join(__dirname, "Photos", `optimized_${i}.jpg`);
return data.toFile(fileUrl);
});
return Promise.all(promises);
};
const doImageManipulation = images => {
// Add your image manipulation here
};
I thought that the resolve() call directly after fs.mkdir is wrong, because mkdir is working asnyc, so the resolve is reached without performing the whole work of mkdir.
const processCompress = new Promise(resolve => {
fs.mkdir(path.join(__dirname, 'Photos', query), function(error) {
let j = 0;
if (!error) {
for (i in urls) {
console.log(i);
var source = tinify.fromUrl(urls[i]);
source.toFile(path.join(__dirname, 'Photos', query, "optimized_" + ++j + ".jpg"));
}
}
resolve(); // <---- inside callback from mkdir.
});
// call the resolve from inside the mkDirs-callback function
// resolve();
});
I hope that will fix your problem.