The following code outputs the content of the index.html (it just contains the text hello world) to the browser. However, when I replace readFile() with readFileSync(), the request times out.
What am I missing? Is a different kind of buffer required? I am using node 0.61 and express 2.4.
var express = require('express');
var fs = require('fs');
var app = express.createServer(express.logger());
app.get('/', function(request, response) {
fs.readFile('index.html', function(err, data){
response.send(data.toString());
});
});
var port = process.env.PORT || 5000;
app.listen(port, function() {
console.log("Listening on " + port);
});
fs.readFile takes a call back which calls response.send as you have shown - good. If you simply replace that with fs.readFileSync, you need to be aware it does not take a callback so your callback which calls response.send will never get called and therefore the response will never end and it will timeout.
You need to show your readFileSync code if you're not simply replacing readFile with readFileSync.
Also, just so you're aware, you should never call readFileSync in a node express/webserver since it will tie up the single thread loop while I/O is performed. You want the node loop to process other requests until the I/O completes and your callback handling code can run.
'use strict'
var fs = require("fs");
/***
* implementation of readFileSync
*/
var data = fs.readFileSync('input.txt');
console.log(data.toString());
console.log("Program Ended");
/***
* implementation of readFile
*/
fs.readFile('input.txt', function (err, data) {
if (err) return console.error(err);
console.log(data.toString());
});
console.log("Program Ended");
For better understanding run the above code and compare the results..
readFileSync() is synchronous and blocks execution until finished. These return their results as return values.
readFile() are asynchronous and return immediately while they function in the background. You pass a callback function which gets called when they finish.
let's take an example for non-blocking.
following method read a file as a non-blocking way
var fs = require('fs');
fs.readFile(filename, "utf8", function(err, data) {
if (err) throw err;
console.log(data);
});
following is read a file as blocking or synchronous way.
var data = fs.readFileSync(filename);
LOL...If you don't want readFileSync() as blocking way then take
reference from the following code. (Native)
var fs = require('fs');
function readFileAsSync(){
new Promise((resolve, reject)=>{
fs.readFile(filename, "utf8", function(err, data) {
if (err) throw err;
resolve(data);
});
});
}
async function callRead(){
let data = await readFileAsSync();
console.log(data);
}
callRead();
it's mean behind scenes readFileSync() work same as above(promise) base.
Related
[EDIT - SOLVED] An important piece of information not in the original question is that this function is running on Zeit Now serverless platform. Zeit functions stop processing after a res.end to keep uptime short. It was confusing because console.logs and other code was being run after the res.end but I assume because it wasn't taking any time to evaluate.
In this simple Node function, why isn't the await function running? I res.end early then want the rest of the function to run but it stops at the await function.
const fs = require("fs")
const bent = require("bent")
const buffer = bent("buffer")
const file_url = "http://klippr.s3.amazonaws.com/assets/mask.png"
module.exports = async (req, res) => {
console.log("REQUEST RECEIVED")
res.end("JOB ACCEPTED")
console.log("STARTING JOB")
console.log("BUT NOTHING AFTER THIS RUNS")
fs.writeFileSync("/tmp/mask.png", await buffer(file_url))
console.log("FILE DOWNLOADED")
// do more stuff with the downloaded file..
}
When I call this function the file is never downloaded and the FILE DOWNLOADED log line doesn't run. Log:
REQUEST RECEIVED
STARTING JOB
BUT NOTHING AFTER THIS RUNS
If I change the fs.writeFileSync to a regular fs.writeFile and use a callback everything works as expected:
const fs = require("fs")
const bent = require("bent")
const buffer = bent("buffer")
const file_url = "http://klippr.s3.amazonaws.com/assets/mask.png"
module.exports = async (req, res) => {
console.log("REQUEST RECEIVED")
res.end("JOB ACCEPTED")
console.log("STARTING JOB")
console.log("BUT NOTHING AFTER THIS RUNS")
// fs.writeFileSync("/tmp/mask.png", await buffer(file_url))
fs.writeFile("/tmp/mask.png", buffer(file_url), () => {
console.log("FILE DOWNLOADED")
})
console.log("FILE DOWNLOADED")
// do more stuff with the downloaded file..
}
Now the file is downloaded and both FILE DOWNLOADED lines log to the console. But I like the sync method because it avoids callback soup.
So my question is why can't an await function run after res.end?
Have you tried to do fs.writeFileSync without awaiting the buffer function like you do in fs.writeFile?
I am trying to make a http request with node module http on an azure function with javascript and for some reason http.request is not receiving data(no error printed its like the requested is blocked). There is anything wrong with azure configuration or the code?Am i missing something really obvious?
Code is running fine on local js file
context.log('JavaScript timer trigger function ran!', timeStamp);
printed as expected but context.log("SUCCESS", JSON.parse(data)); not printed at all.
Also tryed different libraries (request, axios) with no success
module.exports = async function (context, myTimer) {
var timeStamp = new Date().toISOString();
if (myTimer.IsPastDue)
{
context.log('JavaScript is running late!');
}
context.log('JavaScript timer trigger function ran!', timeStamp);
const http = require('http');
http.get('http://myAPIurl', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
context.log("SUCCESS", JSON.parse(data));
});
}).on("error", (err) => {
context.log("ERROR: " + err.message);
});
}
I rewrote your code using Axios. It has support for async/await out of the box, and simplifies a lot of the code to function as your would expect since it makes asynchronous code perform like synchronous code.
I think the main issue you may have been running into it that everything in JavaScript is async. As a result, the Azure Function runtime was exiting before your async function finished since it didn't block like synchronous code would when making the HTTP request. If you want to use callbacks, you need to call context.done() inside your callback function so the Azure Function doesn't exit before the callback is completed. By using async/await, you can guarantee that your code will block on the HTTP request until it receives a response or timeout. In the example below, Axios will return a response object that includes data as an element. I'm extracting data with a deconstruction operation, which allows me to log data.
const axios = require('axios');
const url = 'https://google.com';
module.exports = async function (context, myTimer) {
try {
const { data } = await axios.get(url);
context.log(data);
// do something with the data
return data;
} catch (err) {
context.log(err);
// do something with the error
}
context.done();
}
To Install a Package on an Azure Function
Go to the Azure Portal; select your Azure Function App
Select Platform Features. Then Advanced Tools (Kudu) under Development Tools
Using the file explorer, navigate to site/wwwroot/. You should see a package.json file in there, and several folders each with the name of your functions.
From that directory, run npm install --save axios
To confirm it worked, edit your package.json with the pencil icon. axios should be listed under dependencies json element
I've kept your code callback based.
I removed the async moniker from the definition and added a call to context.done (this signals the functions host when your function has ended) in your resp.end handler
module.exports = function (context, myTimer) {
var timeStamp = new Date().toISOString();
if (myTimer.IsPastDue)
{
context.log('JavaScript is running late!');
}
context.log('JavaScript timer trigger function ran!', timeStamp);
const https = require('https');
https.get('https://raw.githubusercontent.com/LearnWebCode/json example/master/animals-1.json', (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
context.log("SUCCESS", JSON.parse(data));
context.done(null, data);
});
}).on("error", (err) => {
context.log("ERROR: " + err.message);
});
}
Another option would be to keep the function as async but you'd need to replace the callbacks with promise based calls. In some scenarios this can be achieved by wrapping them using util.promisify and then calling them with the await keyword
I have a file, and I want to read it line by line, and for every line extracted I perform some expensive analyzes and then save the results to the database. In short, I have something like this:
const fs = require('fs');
const path = require('path');
const readline = require('readline');
async function analyzeAndSave(url) {
// Removed for brevity, but this function takes a minute or so finsh.
}
async function run() {
try {
const dataPath = path.join(path.dirname(require.main.filename), 'data/urls.txt');
const rl = readline.createInterface({
input: fs.createReadStream(dataPath),
});
let line_no = 0;
rl.on('line', async (url) => {
line_no++;
logger.info(`Analyzing: ${url}`);
await analyzeAndSave(url);
});
} catch (err) {
// Error caught.
logger.error(err);
}
}
run();
The problem with this is that, I notice that it doesn't wait for the analyzes of one line to finish, it kind of tries to execute multiple of the analyzes instances. I can see this as initially it prints all the lines with logger.info('Analyzing: ' + url);`. So, it is not executed sequentially. How can I make sure that one line finishes before moving onto the next?
The readline interface is emitting the "on" events asynchronously and doing an await inside one of them doesn't stop other from being fired. Instead you can buffer up the lines in an array like this:
r.on('line', url => urls.push(url));
r.on('close', async () => {
for (const url of urls) {
await analyzeAndSave(url);
}
});
where urls is initialized to an empty array before the readline interface is created.
I think this is going to be helpful to you, exampled and mentioned here.
Nodejs - read line by line from file, perform async action for each line and reusme
Someone stated you can use a library for big files which is titled: line-by-line
#JavierFerrero stated a solution as such.
var LineByLineReader = require('line-by-line'),
lr = new LineByLineReader('big_file.txt');
lr.on('error', function (err) {
// 'err' contains error object
});
lr.on('line', function (line) {
// pause emitting of lines...
lr.pause();
// ...do your asynchronous line processing..
setTimeout(function () {
// ...and continue emitting lines.
lr.resume();
}, 100);
});
lr.on('end', function () {
// All lines are read, file is closed now.
});
You can also pass it ass a callback, waiting for the operation to
finish.
const fs = require('fs');
function run(path, cb) {
try {
fs.readFile(path, 'utf8', function(err, data){
if(err) throw err;
cb(data);
});
} catch (err) {
// Error caught.
}
}
run('./test.txt', (response) => {
// We are done, now continue
console.log(response)
})
I get an issue where If I try to use request more than once, it doesn't work. I can give an example:
request('https://www.google.com', function (error, response, vBody1) {
request('https://www.purple.com', function (error, response, vBody2) {
request('https://www.stackoverflow.com', function (error, response, vBody3) {
console.log(vBody3);
});
console.log(vBody2);
});
console.log(vBody1);
});
It reads and outputs the body of the first site, but not the next two.
Here is another example:
vSites = ['https://www.google.com','https://www.purple.com','https://www.stackoverflow.com'];
for (i = 0; i < vSites.length; i++){
request(vSites[i], function (error, response, vBody[i]) {
console.log(vBody[i]);
});
}
This one looks cleaner and I would like this version, but it doesn't work at all. What am I doing wrong?
In your loop version you don't need the index on the vBody variable.
var request = require('request');
vSites = ['https://www.google.com','https://www.purple.com','https://www.stackoverflow.com'];
for (i = 0; i < vSites.length; i++){
request(vSites[i], function (error, response, vBody) {
console.log(vBody);
});
}
This code works fine for me on node v4.3.2
In addition to the correct solution from Michael's reply, please be aware the for-loop sends the requests in parallel. Data and error handling could be challenging if you want to do more than console.log.
To better handle the requests, I would suggest using an asynchronous control library such as async or bluebird. In comparison, async is easier to understand while bluebird provides better code readability.
Below is an example async implementation:
var async = require('async');
var request = require('request');
var vSites = ['https://www.google.com', 'http://www.purple.com','https://www.stackoverflow.com'];
async.map(
vSites,
// `async` will call this function for each `vSite`
function(vSite, next) {
// Reuse the code inside your for-loop, but call `next` to pass the error and result to the final callback
request(vSite, function(err, response, vBody) {
next(err, vBody);
});
},
// The final callback will be executed when all requests are completed or either of them fails
function(err, results) {
console.log(err);
console.log(results);
}
);
Hope this helps :)
I have a config json file as below.
"constraints": {
"input": "input",
"output": "output"
}
I am trying to read this file and create input and output directory with child directories.
var fs = require("fs");
var async = require("async");
var node_xj = require("xls-to-json");
var cf= JSON.parse(fs.readFileSync("config.json", 'utf8'));
// Declare variables
var files = [];
function readAsync(file, callback) {
node_xj(file, callback);
}
function create(currentDirPath, outputDirPath, callback) {
// some code creating directories
}
create(cf.lang.input, cf.lang.output, function(stat, config) {
files.push(config);
});
async.map(files, readAsync, function(err, results) {
if(err) throw err;
});
The code works fine, but its sometimes does not. Let me walk through the code. I am loading some modules.
var fs = require("fs");
var async = require("async");
var node_xj = require("xls-to-json");
I am loading the config file.
var cf= JSON.parse(fs.readFileSync("config.json", 'utf8'));
Then i am passing the cf file to create function which after its operation returns me an an object which i push it into an array.
var files = [];
function readAsync(file, callback) {
node_xj(file, callback);
}
function create(input, output, callback) {
// some code creating directories and object
}
create(cf.lang.input, cf.lang.output, function(stat, config) {
files.push(config);
});
async.map(files, readAsync, function(err, results) {
if(err) throw err;
});
Then i am passing the files array to my async.map function which passes it to readAsync function for another operation.
Question:
Can anyone tell me whether the way i have written the code flow
makes if flaw sometimes.
Is there a better way of writing the same code logic flow.
Should i use async.map to iterate files and then pass it to
readAsync
If you're depending on asynchronous operations to occur before other operations, you have to treat them asynchronously! Your code as described seems to create some directories (which will take measurable time) and immediately attempts to use them, which isn't kosher.
You might consider something like:
async.series([
function(){
// create your directories here
},
function(){
async.map(files,...)
}
]);
which will guarantee that the resources needed by your map exist before the map is called.