I would like to read a remote image and display it. I can save the file but not getting the code right to display it. Ideally I just want to pass the file right though without processing - not sure if a tmp file step is required or not. This code displays nothing - no errors. I tried res.pipe(response) as well.
var url = 'http://proxy.boxresizer.com/convert?resize=50x50&source=' + filename
var request = http.get(url, function(response) {
var tmp = path.join(require('os').tmpDir(), filename);
var outstream = require('fs').createWriteStream(tmp);
response.pipe(outstream);
response.on('end', function() {
res.set('Content-Type', 'image/jpg');
res.pipe(outstream);
res.end();
});
});
Well I'd still like to know how to make the above work but I solved my issue in one line with the request module!
var url = 'http://proxy.boxresizer.com/convert?resize=50x50&source=' + filename
require('request').get(url).pipe(res); // res being Express response
Since request is deprecated, you can alternatively use node-fetch like so:
app.get("/", (req, res) => {
fetch(actualUrl).then(actual => {
actual.headers.forEach((v, n) => res.setHeader(n, v));
actual.body.pipe(res);
});
});
This is what seems to work for me, using the standard fetch() included in Node 18, which is a bit different than node-fetch, see github.com/node-fetch/node-fetch#bodybody:
app.get("/", (req, res) => {
fetch(url).then((response) => {
body.pipeTo(
new WritableStream({
start() {
response.headers.forEach((v, n) => res.setHeader(n, v));
},
write(chunk) {
res.write(chunk);
},
close() {
res.end();
},
})
);
});
});
Related
i'm trying to write a feed to a file using node.js. the problem is, it doesn't write all the feeds, only the last 1.
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
feedParser.parseUrl(url).on('article', function(article) {
console.log('title; ', article.title);
fs.writeFile("articles.json", JSON.stringify(article.title), function(err) {
if(err) {
console.log(err);
}
});
});
Why?
Just change fs.writeFile( to fs.appendFile( and you're fine.
fs.writeFile overwrites your file each time you call it whereas fs.appendFile adds to a file.
As #Robert says you should use appendFile, but also note that that change won't write out valid json. I'm not sure what output you're trying to achieve - it you just want the titles you could write out a txt file with a title on each line like so:
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
feedParser.parseUrl(url).on('article', function(article) {
console.log('title; ', article.title);
fs.appendFile("articles.txt", article.title + "\n", function(err) {
if(err) {
console.log(err);
}
});
});
To write out json you can do:
var fs = require('fs');
var feedParser = require('ortoo-feedparser')
var url = "http://iwnsvg.com/feed";
let titles = [];
feedParser.parseUrl(url)
.on('article', function (article) {
console.log('title; ', article.title);
titles.push(article.title);
})
.on('end', function () {
fs.writeFile('articles.json', JSON.stringify({ titles }), function (err) {
if (err) {
console.log(err);
}
});
});
fs.writeFile comes with some options like flag. Default value of flag is w for write, so your data are replaced by the new one.
Use 'a' instead
{flag:'a'}
and you'll be fine.
But don't forget that WriteFile or AppendFile are upper layer in fs library which open and close file each time you need to add data.
Preferably, use fs.createWriteStream which returns a writable stream (writable file handle in other languages). Then use and reuse this stream when you need to write data in your file.
This is a Node application, running express server. I have a folder with text files in it. I need to be able to go into each one of those files inside the folder, and extract lines that include a word "SAVE".
I am stuck at this step.
app.get('/logJson', function (req, res) {
const logsFolder = 'C:/logs/';
fs.readdir(logsFolder, (err, files) => {
if (err) {
res.send("[empty]");
return;
}
files.forEach(function(filename){
var logFiles = fs.readFileSync (logsFolder + filename, 'ascii').toString().split("\n");
console.log(logFiles + "\n");
})
})
})
I cannot figure out where do I include this:
var log = (logFiles.split(/.*SAVE.*/g)||['no result found'])[0];
Any help would be appreciated
If you only want to print out all lines containing the word SAVE, you could do it like this.
Remark: I did not run this code.
app.get('/logJson', function (req, res) {
const logsFolder = 'C:/logs/';
fs.readdir(logsFolder, (err, files) => {
if (err) {
res.send("[empty]");
return;
}
var lines = [];
files.forEach(function(filename) {
var logFileLines = fs.readFileSync (logsFolder + filename, 'ascii').toString().split("\n");
// go through the list of logFileLines
logFileLines.forEach(function(logFileLine) {
// if the current line matches SAVE, it will be stored in the variable lines
if(logFileLine.match(/SAVE/)) {
lines.push(logFileLine);
}
})
})
// the variable lines is printed to the console
console.log(lines);
})
})
Recently I started learning a little bit about Node.js and it's capabilities and tried to use it for some web services.
I wanted to create a web service which will serve as a proxy for web requests.
I wanted my service to work that way:
User will access my service -> http://myproxyservice.com/api/getuserinfo/tom
My service will perform request to -> http://targetsite.com/user?name=tom
Responded data would get reflected to the user.
To implement it I used the following code:
app.js:
var express = require('express');
var bodyParser = require('body-parser');
var app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
var proxy = require('./proxy_query.js')
function makeProxyApiRequest(name) {
return proxy.getUserData(name, parseProxyApiRequest);
}
function parseProxyApiRequest(data) {
returned_data = JSON.parse(data);
if (returned_data.error) {
console.log('An eror has occoured. details: ' + JSON.stringify(returned_data));
returned_data = '';
}
return JSON.stringify(returned_data);
}
app.post('/api/getuserinfo/tom', function(request, response) {
makeProxyApiRequest('tom', response);
//response.end(result);
});
var port = 7331;
proxy_query.js:
var https = require('https');
var callback = undefined;
var options = {
host: 'targetsite.com',
port: 443,
method: 'GET',
};
function resultHandlerCallback(result) {
var buffer = '';
result.setEncoding('utf8');
result.on('data', function(chunk){
buffer += chunk;
});
result.on('end', function(){
if (callback) {
callback(buffer);
}
});
}
exports.getUserData = function(name, user_callback) {
callback = user_callback
options['path'] = user + '?name=' + name;
var request = https.get(options, resultHandlerCallback);
request.on('error', function(e){
console.log('error from proxy_query:getUserData: ' + e.message)
});
request.end();
}
app.listen(port);
I wish I didn't screwed this code because I replaced some stuff to fit my example.
Anyway, the problem is that I want to post the response to the user when the HTTP request is done and I cant find how to do so because I use express and express uses asynchronous calls and so do the http request.
I know that if I want to do so, I should pass the makeProxyApiRequest the response object so he would be able to pass it to the callback but it is not possible because of asyn problems.
any suggestions?
help will be appreciated.
As you're using your functions to process requests inside your route handling, it's better to write them as express middleware functions, taking the specific request/response pair, and making use of express's next cascade model:
function makeProxyApiRequest(req, res, next) {
var name = parseProxyApiRequest(req.name);
res.locals.userdata = proxy.getUserData(name);
next();
}
function parseProxyApiRequest(req, res, next) {
try {
// remember that JSON.parse will throw if it fails!
data = JSON.parse(res.locals.userdata);
if (data .error) {
next('An eror has occoured. details: ' + JSON.stringify(data));
}
res.locals.proxyData = data;
next();
}
catch (e) { next("could not parse user data JSON."); }
}
app.post('/api/getuserinfo/tom',
makeProxyApiRequest,
parseProxyApiRequest,
function(req, res) {
// res.write or res.json or res.render or
// something, with this specific request's
// data that we stored in res.locals.proxyData
}
);
Even better would be to move those middleware functions into their own file now, so you can simply do:
var middleware = require("./lib/proxy_middleware");
app.post('/api/getuserinfo/tom',
middleware.makeProxyApiRequest,
middleware.parseProxyApiRequest,
function(req, res) {
// res.write or res.json or res.render or
// something, with this specific request's
// data that we stored in res.locals.proxyData
}
);
And keep your app.js as small as possible. Note that the client's browser will simply wait for a response by express, which happens once res.write, res.json or res.render etc is used. Until then the connection is simply kept open between the browser and the server, so if your middleware calls take a long time, that's fine - the browser will happily wait a long time for a response to get sent back, and will be doing other things in the mean time.
Now, in order to get the name, we can use express's parameter construct:
app.param("name", function(req, res, next, value) {
req.params.name = value;
// do something if we need to here, like verify it's a legal name, etc.
// for instance:
var isvalidname = validator.checkValidName(name);
if(!isvalidname) { return next("Username not valid"); }
next();
});
...
app.post("/api/getuserinfo/:name", ..., ..., ...);
Using this system, the :name part of any route will be treated based on the name parameter we defined using app.param. Note that we don't need to define this more than once: we can do the following and it'll all just work:
app.post("/api/getuserinfo/:name", ..., ..., ...);
app.post("/register/:name", ..., ..., ... );
app.get("/api/account/:name", ..., ..., ... );
and for every route with :name, the code for the "name" parameter handler will kick in.
As for the proxy_query.js file, rewriting this to a proper module is probably safer than using individual exports:
// let's not do more work than we need: http://npmjs.org/package/request
// is way easier than rolling our own URL fetcher. In Node.js the idea is
// to write as little as possible, relying on npmjs.org to find you all
// the components that you need to glue together. If you're writing more
// than just the glue, you're *probably* doing more than you need to.
var request = require("request");
module.exports = {
getURL: function(name, url, callback) {
request.get(url, function(err, result) {
if(err) return callback(err);
// do whatever processing you need to do to result:
var processedResult = ....
callback(false, processedResult);
});
}
};
and then we can use that as proxy = require("./lib/proxy_query"); in the middleware we need to actually do the URL data fetching.
I am trying to write the binary body of a request to a file and failing. The file is created on server but I am unable to open it. I am getting 'Fatal error: Not a png' on Ubuntu. Here is how I am making the request:
curl --request POST --data-binary "#abc.png" 192.168.1.38:8080
And here is how I am trying to save it with the file. The first snippet is a middleware for appending all the data together and second one is the request handler:
Middleware:
app.use(function(req, res, next) {
req.rawBody = '';
req.setEncoding('utf-8');
req.on('data', function(chunk) {
req.rawBody += chunk;
});
req.on('end', function() {
next();
});
});
Handler:
exports.save_image = function (req, res) {
fs.writeFile("./1.png", req.rawBody, function(err) {
if(err) {
console.log(err);
} else {
console.log("The file was saved!");
}
});
res.writeHead(200);
res.end('OK\n');
};
Here's some info which might help. In the middleware, if I log the length of rawBody, it looks to be correct. I am really stuck at how to correctly save the file. All I need is a nudge in the right direction.
Here is a complete express app that works. I hit it with curl --data-binary #photo.jpg localhost:9200 and it works fine.
var app = require("express")();
var fs = require("fs");
app.post("/", function (req, res) {
var outStream = fs.createWriteStream("/tmp/upload.jpg");
req.pipe(outStream);
res.send();
});
app.listen(9200);
I would just pipe the request straight to the filesystem. As to your actual problem, my first guess is req.setEncoding('utf-8'); as utf-8 is for text data not binary data.
For fix your code: I'm with #Peter Lyons, that the error is probably the req.setEncoding('utf-8'); line.
I know the following don't ask your question directly, but proposes an alternative to it by using req.files functionality provided by Express.js, which you are using.
if (req.files.photo && req.files.photo.name) {
// Get the temporary location of the file.
var tmp_path = req.files.photo.path;
// set where the file should actually exists - in this case it is in the "images" directory.
var target_path = './public/profile/' + req.files.photo.name;
// Move the file from the temporary location to the intended location.
fs.rename(tmp_path, target_path, function (error) {
if (!error) {
/*
* Remove old photo from fs.
* You can remove the following if you want to.
*/
fs.unlink('./public/profile/' + old_photo, function () {
if (error) {
callback_parallel(error);
}
else {
callback_parallel(null);
}
});
}
else {
callback_parallel(error);
}
});
}
var express = require('express')
var app = module.exports = express()
function getImages(callback) {
callback()
}
app
.set('views', __dirname + '/views')
.set('view engine', 'jade')
.get('/stream/images', function(req, res) {
res.render('layout', function(err, body) {
res.type('html')
res.write(body.replace('</body></html>', ''))
getImages(function(err) {
res.render('streams/images', function(err, body) {
var html = '<div class="layout-stream">' + body + '</div>'
res.write(
"<script>$('.pagelet-layout-stream').replaceWith(" +
JSON.stringify(html) +
"); emitter.emit('check:stream');</script>"
)
res.write("<script>emitter.emit('load');</script>")
res.write('</body></html>')
res.end()
})
})
})
})
.listen(3013)
This works exactly as intended.
However, when I .use(express.compress()) (before .get to be exact), the page no longer streams.
In other words, the express.compress() seems to wait until the response is finished, then gzip the entire response, then send it.
I would like each res.write to send a gzipped response (or specifically, every drain).
How can I compress the response correctly?
Edit 1 - I tried this:
var stream = zlib.createGzip()
stream.pipe(res)
stream.write(/* stuff */)
stream.end()
This works exactly like express.compress(). I'm not sure whether the browser just doesn't parse the body until it has the whole thing, or it doesnt send the body until the response is complete.
var stream = zlib.createGzip()
stream._flush = zlib.Z_SYNC_FLUSH
stream.pipe(res)
stream.write(/* stuff */)
stream.end()