Using Node.js to upload S3 object to FTP server - javascript

I need a Node.js script that does the following:
1 - Triggers when an image is added to a specified S3 bucket.
2 - Creates a thumbnail of that image (360x203 pixels).
3 - Saves a copy of that thumbnail inside of a separate S3 folder.
4 - Uploads the thumbnail to a specified FTP server, SIX (6) times using a "FILENAME-X"naming convention.
The problem: I am not able to get the c.connect or c.append to work. I have tried about everything, and scoured the internet. It seems like it should work, but just doesn't.
Note: I removed my FTP credentials for privacy.
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');
var Client = require('ftp');
var fs = require('fs');
var gm = require('gm')
.subClass({ imageMagick: true }); // Enable ImageMagick integration.
// get reference to FTP client
var c = new Client();
// get reference to S3 client
var s3 = new AWS.S3();
exports.handler = function(event, context) {
// Read options from the event.
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
// Get source bucket
var srcBucket = event.Records[0].s3.bucket.name;
// Get source object key
// Object key may have spaces or unicode non-ASCII characters.
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var url = 'http://' + srcBucket + ".s3.amazonaws.com/" + srcKey;
// Set destination bucket
var dstBucket = srcBucket + "-thumbs";
// Set destination object key
var dstKey = "resized-" + srcKey;
// Infer the image type.
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
console.error('unable to infer image type for key ' + srcKey);
return;
}
var imageType = typeMatch[1];
if (imageType != "jpg" && imageType != "png") {
console.log('skipping non-image ' + srcKey);
return;
}
// Download the image from S3, transform, and upload to a different S3 bucket.
async.waterfall([
function download(next) {
// Download the image from S3 into a buffer.
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
gm(response.Body).size(function(err, size) {
// Transform the image buffer in memory.
this.toBuffer(imageType, function(err, buffer) {
if (err) {
next(err);
} else {
next(null, response.ContentType, buffer);
}
});
});
},
function upload(contentType, data, next) {
// Connect to server
c.connect({
host: "localhost",
port: 21, // defaults to 21
user: "", // defaults to "anonymous"
password: "", // defaults to "#anonymous"
});
// Upload test file to FTP server
c.append(data, srcKey, function(err) {
console.log("CONNECTION SUCCESS!");
if (err) throw err;
c.end();
});
// Stream the thumb image to a different S3 bucket.
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: data,
ContentType: contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to resize ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully resized ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
context.done();
}
);
};

Related

How to close Busboy connection and send back an error response in ExpressJS

I have busboy in my ExpressJS app for file streaming. Now at some point I wish to send back a regular response that the file size is too large. How can I do that?
Here is my code to illustrate what I want:
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
let fileSize = 0;
file.on('data', function(data) {
fileSize += data.length;
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
if (fileSize > MAXIMUM_ALLOWED_FILE_UPLOAD_SIZE) {
//I wish to send back a regular response that the file size is too large here.
}
fileSize = 0;
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
});
How can I achieve that?
Busboy has a maximum file byte size that you can specify in the limits option which will prevent it from pushing any more data into the file than your specified limit. If you want to limit your file upload size, I'd go with that. It emits a 'limit' event on the file if it reaches the maximum size.
Busboy also handles multiple uploads-- not sure if you wanted to handle that or not. But here's a sample with a limit and multiple uploads:
var express = require('express');
var Busboy = require('busboy');
var html_escaper = require('html-escaper');
var fs = require('fs');
var escaper = html_escaper.escape;
const MAX_UPLOAD_SIZE = 10000;
var app = express();
app.get('/', function(req, res) {
res.writeHead(200, {'Content-type': 'text/html'})
res.write('<!doctype html><html><body>');
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">')
res.write('<input type="file" name="upload_1">')
res.write('<input type="file" name="upload_2">');
res.write('<input type="submit"></form>');
res.write('</body></html>')
return res.end();
})
app.post('/fileupload', function (req, res) {
var busboy = new Busboy({ headers: req.headers, limits: { fileSize: MAX_UPLOAD_SIZE } });
var fileList = [];
busboy.on('file', function(fieldname, file, filename, encoding, mimetype ) {
fileInfo = {
'fObj': file,
'file': filename,
'status': 'uploaded'
}
fileList.push(fileInfo);
file.on('limit', function() {
fileInfo.status = `Size over max limit (${MAX_UPLOAD_SIZE} bytes)`;
});
file.on('end', function() {
if (fileInfo.fObj.truncated) {
if (fileInfo.status === 'uploaded') {
// this shouldn't happen
console.error(`we didn't get the limit event for ${fObj.file}???`);
fileInfo.status = `Size over max limit (${MAX_UPLOAD_SIZE} bytes)`;
}
}
});
file.pipe(fs.createWriteStream(`./uploads/${fileList.length}.bin`));
});
busboy.on('finish', function() {
var out = "";
fileList.forEach(o => {
out += `<li>${escape(o['file'])} : ${o['status']}</li>`
});
res.writeHead(200, {'Connection': 'close'});
res.end(`<html><body><ul>${out}</ul></body></html>`);
})
return req.pipe(busboy);
});
app.listen(8080);

Aws4 sign S3 PUT requests

Writing specialized S3 file upload request signing function that will run on Cloudflare workers (I guess should be the same as in browsers):
let s3PutSign = function(region, keyId, keySecret, contentType, date, bucket, fileName) {
return crypto.subtle.importKey('raw', new TextEncoder().encode(keySecret), { name: 'HMAC', hash: 'SHA-256' }, true, ['sign'])
.then(key => {
let path = `/${bucket}/${fileName}`
let strToSign = `PUT\n\n${contentType}\n${date}\n${path}`
return crypto.subtle.sign('HMAC', key, new TextEncoder().encode(strToSign))
.then(sig => {
return {
url: `https://s3.${region}.amazonaws.com${path}`,
headers: {
'content-type': contentType,
'Authorization': `AWS ${keyId}:${btoa(sig)}`,
'x-amz-date': new Date(new Date().getTime() + 10000).toISOString().replace(/[:\-]|\.\d{3}/g, '').substr(0, 17)
}
}
})
})
}
Wrote function using PUT example: https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html
Variable strToSign:
PUT
application/pdf
Wed, 27 May 2020 12:26:33 GMT
/mybucket/file.pdf
function result:
{
url: "https://s3.eu-central-1.amazonaws.com/mybucket/file.pdf",
headers: {
content-type: "application/pdf",
Authorization: "AWS AKXAJE7XIIVXQZ4X7FXQ:W29iamVXZCBBcnJheUJ1ZmZlcl0=",
x-amz-date: "20200527T122643Z"
}
}
Requests always result this response:
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>InvalidRequest</Code>
<Message>The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.</Message>
<RequestId>7CECC87D5E855C48</RequestId>
<HostId>rtGLR0u9Qc29bllgKnJf7xD00iQ0+/BZog5G/wYWjsN8tkXio9Baq7GZvbQTD40EVCQ9FzuCo9c=</HostId>
</Error>
Please advise how to debug or give a hint what could be wrong with this function.
Researching this a bit, it seems that AWS4-HMAC-SHA256 may define a specific hashing algorithm. Looking at this (awesome) gist, the author calls out the full algo name.
You might try replacing your call ({ name: 'HMAC', hash: 'SHA-256' }) with { name: 'HMAC', hash: 'AWS4-HMAC-SHA256' }
Another thought is to remove the dash (-) from your algorithm name. Go from SHA-256 to SHA256 and see if that makes a diff.
I know the post is not new, but maybe it will help someone else. :)
Here is a working code example, that I gathered from AWS documentation and some other sources.
// make the call
let signer = new S3Signer(body,'ACCESKEYS','SECRETACCESSKEY','us-east-1','my-test-bucket23.s3.us-east-1.amazonaws.com','/something/SOME/2023/02/some6.pdf','multipart/form-data');
let signerRes = await signer.makeTheCall();
// implementation
const axios = require('axios');
const crypto = require('crypto');
class S3Signer {
constructor(body, accessKey, secretKey, region, host, path, contentType) {
this.region = region; //'us-east-1';
this.host = host; // `my-test-bucket23.s3.us-east-1.amazonaws.com`
this.method = 'PUT';
this.service = 's3';
this.path = path; // `/something/SOME/2023/02/some6.pdf`
this.url = `https://${this.host}${this.path}`;
this.contentType = contentType; //'multipart/form-data';
this.amzDate = null;
this.body = body;
//this.bodyUTF8 = body.toString('utf8');
this.algorithm = 'AWS4-HMAC-SHA256';
this.credentialScope = '';
this.signedHeaders = 'content-type;host;x-amz-content-sha256;x-amz-date';
this.accessKey = accessKey;
this.secretKey = secretKey;
}
getSignatureKey = (key, dateStamp, regionName, serviceName) => {
let kDate = this.hmac(('AWS4' + key), dateStamp);
let kRegion = this.hmac(kDate, regionName);
let kService = this.hmac(kRegion, serviceName);
let kSigning = this.hmac(kService, 'aws4_request');
return kSigning;
}
makeTheCall = async () => {
let canonicalRequest = this.createCanonicalReq();
let signature = this.calculateSignature(canonicalRequest);
// ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *************
// Put the signature information in a header named Authorization.
let authorizationHeader = this.algorithm + ' ' + 'Credential=' + this.accessKey + '/' + this.credentialScope + ', ' + 'SignedHeaders=' + this.signedHeaders + ', ' + 'Signature=' + signature;
// For DynamoDB, the request can include any headers, but MUST include "host", "x-amz-date",
// "x-amz-target", "content-type", and "Authorization". Except for the authorization
// header, the headers must be included in the canonical_headers and signed_headers values, as
// noted earlier. Order here is not significant.
//// Python note: The 'host' header is added automatically by the Python 'requests' library.
let headers = {
'Authorization': authorizationHeader,
'Content-Type': this.contentType,
'X-Amz-Content-Sha256' : this.hash(this.body, 'hex'),
'X-Amz-Date': this.amzDate
}
let request = {
host: this.host,
method: this.method,
url: this.url,
data: this.body,
body: this.body,
path: this.path,
headers: headers
}
// send the file to s3
let res = await axios(request);
console.log(res);
return res;
}
calculateSignature = (canonicalReq) => {
// SHA-256 (recommended)
let dateStamp = this.amzDate.substring(0, 8);
this.credentialScope = dateStamp + '/' + this.region + '/' + this.service + '/' + 'aws4_request'
let stringSign = this.algorithm + '\n' + this.amzDate + '\n' + this.credentialScope + '\n' + this.hash(canonicalReq, 'hex')
// ************* TASK 3: CALCULATE THE SIGNATURE *************
// Create the signing key using the function defined above.
let signingKey = this.getSignatureKey(this.secretKey, dateStamp, this.region, this.service);
// Sign the string_to_sign using the signing_key
let signature = this.hmac(signingKey, stringSign, 'hex');
return signature;
}
createCanonicalReq = () => {
// ************* TASK 1: CREATE A CANONICAL REQUEST *************
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// Step 1 is to define the verb (GET, POST, etc.)--already done.
// Step 2: Create canonical URI--the part of the URI from domain to query
// string (use '/' if no path)
let canonical_uri = this.path;
//// Step 3: Create the canonical query string. In this example, request
// parameters are passed in the body of the request and the query string
// is blank.
let canonical_querystring = '';
///// set the date
let date = new Date();
this.amzDate = date.toISOString().replace(/[:\-]|\.\d{3}/g, '')
// Step 4: Create the canonical headers. Header names must be trimmed
// and lowercase, and sorted in code point order from low to high.
// Note that there is a trailing \n.
let canonical_headers = 'content-type:' + this.contentType + '\n' + 'host:' + this.host + '\n'
+ 'x-amz-content-sha256:' + this.hash(this.body, 'hex') + '\n'
+ 'x-amz-date:' + this.amzDate + '\n'
// Step 5: Create the list of signed headers. This lists the headers
// in the canonical_headers list, delimited with ";" and in alpha order.
// Note: The request can include any headers; canonical_headers and
// signed_headers include those that you want to be included in the
// hash of the request. "Host" and "x-amz-date" are always required.
// For DynamoDB, content-type and x-amz-target are also required.
//this.signedHeaders = 'content-type;host;x-amz-date'
// Step 6: Create payload hash. In this example, the payload (body of
// the request) contains the request parameters.
let payload_hash = this.hash(this.body, 'hex');
// Step 7: Combine elements to create canonical request
let canonicalRequest = this.method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + this.signedHeaders + '\n' + payload_hash
return canonicalRequest;
}
hmac = (key, string, encoding) => {
return crypto.createHmac('sha256', key).update(string, 'utf8').digest(encoding)
}
hash = (string, encoding) => {
return crypto.createHash('sha256').update(string, 'utf8').digest(encoding)
}
// This function assumes the string has already been percent encoded
// only if you have reuquest query
encodeRfc3986 = (urlEncodedString) => {
return urlEncodedString.replace(/[!'()*]/g, (c) => {
return '%' + c.charCodeAt(0).toString(16).toUpperCase()
})
}
}

Doesn't appear the actual error through command line in Node.js

I am trying to create an HTTP server with GET and POST, using Node.js, the user will upload images and archives. But the problem is that while trying to run the server, the command line from Node.js don't show to me the actual error, and point out to some line that doesn't exist.
Server.js
var url = require("url");
var http = require("http");
var formidable = require("formidable");
function start(route ,handle) {
function onRequest(request, response) {
var postData = "";
var pathname = url.parse(request.url).pathname;
console.log("Request for " + pathname + " received.");
request.setEncoding("utf8");
request.addListener("data", function(postDataChunk) {
postData += postDataChunk;
console.log("Received POST data chunk ' " +
postDataChunk + " ' .");
});
request.addListener("data", function(chunk) {
//called when a new chunk of data was received
});
request.addListener("end", function() {
//called when all chunks of data been received
});
request.addListener("end", function() {
route(handle, pathname, response, postData);
});
}
var formidable = require('formidable'),
http = require('http'),
sys = require('sys');
http.createServer(function(req,res) {
if(req.url == '/upload' && req.method.toLowerCase() == 'POST') {
//parse a file upload
var form = new formidable.IncomingForm();
form.parse(req,function(err,fields,files) {
res.writeHead(200, {'content-type' : 'text/plain'});
res.write('received upload \n\n');
res.end(sys.inspect({fields : fields, files : files}));
});
return;
}
//show a file upload form
res.writeHead(200, {'content-type' : 'text/html'});
res.end(
'<form action = "/upload" enctype="multipart/form-data" ' +
'method ="post">' +
'<input type="text" name= "title" ><br>' +
'<input type ="file" name = "upload" multiple="multiple"<br>' +
'<input typw="submit" value="upload"' +
'</form>');
}).listen(8888);
exports.start = start;
The only thing for sure is that I know that is happening in Server.js because is pointed out to there.
server.js:69 - Unexpected token )
How can I get to know where this error is ? And maybe can be a lot of other sintax errors.
You're missing the closing } for the start function. Try adding that in and see if the error goes away. This is how your code should look (with correct indents to make it easier to spot errors):
var url = require("url");
var http = require("http");
var formidable = require("formidable");
function start(route ,handle) {
function onRequest(request, response) {
var postData = "";
var pathname = url.parse(request.url).pathname;
console.log("Request for " + pathname + " received.");
request.setEncoding("utf8");
request.addListener("data", function(postDataChunk) {
postData += postDataChunk;
console.log("Received POST data chunk ' " +
postDataChunk + " ' .");
});
request.addListener("data", function(chunk) {
//called when a new chunk of data was received
});
request.addListener("end", function() {
//called when all chunks of data been received
});
request.addListener("end", function() {
route(handle, pathname, response, postData);
});
}
var formidable = require('formidable'),
http = require('http'),
sys = require('sys');
http.createServer(function(req,res) {
if(req.url == '/upload' && req.method.toLowerCase() == 'POST') {
//parse a file upload
var form = new formidable.IncomingForm();
form.parse(req,function(err,fields,files) {
res.writeHead(200, {'content-type' : 'text/plain'});
res.write('received upload \n\n');
res.end(sys.inspect({fields : fields, files : files}));
});
return;
}
//show a file upload form
res.writeHead(200, {'content-type' : 'text/html'});
res.end(
'<form action = "/upload" enctype="multipart/form-data" ' +
'method ="post">' +
'<input type="text" name= "title" ><br>' +
'<input type ="file" name = "upload" multiple="multiple"<br>' +
'<input typw="submit" value="upload"' +
'</form>');
}).listen(8888);
}
exports.start = start;

Edit JSON file through loop in node js

I am making a monitor which will display to a website if a server is up or down. I have tried numerous things including regex, replacement, etc. when it comes to editing the JSON file. I need the "status" in the JSON file to adjust to up or down based upon the 200 responseCode(the status=up/down is just a placeholder for now). Every time I am able to append the JSON file in anyway the text is always added to the bottom of the page outside of the object. It is important to note that this JSON file is used for the front end in the way that displays a green block if the status is 'up', red if it is 'down.' There are also 11 severs in total in my original JSON file.
const config = require(__dirname + '/config/config.json');
var fs = require('fs');
...
function wrapper() {
//FOR EACH SERVER IN CONFIG FILE
for (var server of config.server) {
var url = server.ip + ':' + server.port;
var name = url + " " + server.name;
// DO REQUEST
loadUrl(url, name);
}
}
function loadUrl(url, name) {
var start = new Date().getTime();
request(url, function(error, response) {
var logName = name.split(' ');
var end = new Date().getTime();
var time = end - start;
if (!error) {
console.log(DateTime + " " + name + " " + response.statusCode + " - response time: " + time + "ms");
var total = DateTime + " " + name + " " + response.statusCode + " - response time: " + time + "ms\n";
fs.appendFile("./BY-logs/" + logDates + "_" + logName[1] + ".txt", total, encoding='utf8', function (err) { if (err) throw err; }); //creating or adding to logfile
//CHANGE JSON ACCORDING TO STATUS
if( response.statusCode != 200) {
fs.appendFile('./config/config.json', status = down, function (err) {});
} else {
fs.appendFile('./config/config.json', status = up, function (err) {});
}
}
}
)};
wrapper();
SAMPLE FROM JSON FILE:
{
"port": 3000,
"server": [{
"id": 1,
"name": "abcdefghi2(node-1)",
"ip": "http://123.123.123.12",
"port": "8080",
"status": "up"
}, {
"id": 2,
"name": "abcdefg(node-2)",
"ip": "http://123.123.123.13",
"port": "8080",
"status": "up"
}]
}
You need to load the JSON into the program, change the value of the object, then write it to a file.
Try something along these lines:
var config = require('./config/config.json');
config.server.forEach((server, index, array) => {
if (server.id == "x")
server.status = "up"; // etc.
});
fs.writeFile('./config/config.js', config, (err) => { /* ... */ });
Side note: I would recommened storing your servers as objects inside that object, not an array of objects. This makes it easier to get the server object you want.

selenium screenshots not saving specified directory

I have my selenium test set up to take screenshots, but they are not saving to the directory which I have specified. Can anybody show me what I am missing?
Here is how I am configuring the screenshots in the test:
function writeScreenshot(data, name) {
var fs = require('fs');
name = name || 'ss.png';
var screenshotPath = mkdirp(configuration.readSettings('screenshotDirectory') + fileNameURL + "/", function(err){});
fs.writeFileSync(screenshotPath + name, data, 'base64');
};
and then I take the screenshot:
driver.takeScreenshot().then(function(data) {
var screenshotFile = os + '_' + osVersion + '_' + browser + '_' + browserVersion + '.png';
writeScreenshot(data, screenshotFile);
});
The screenshots end up being saved instead in the projects root directory and with the file name preceded by 'undefined'. (ex. undefinedWindows_8_chrome_46.png)
It does, however, create the folders shown here: var screenshotPath = mkdirp(configuration.readSettings('screenshotDirectory') + fileNameURL + "/", function(err){});
So why is this happening?
mkdirp() is an async method. That is why you pass a callback. You will need to change your code to something like the following:
function writeScreenshot(data, name) {
var fs = require('fs');
name = name || 'ss.png';
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
mkdirp(screenshotPath, function(err){
if (err) {
// something else happened while creating the dir. You decide what to do
return;
}
// Otherwise (if dir was created)
fs.writeFileSync(screenshotPath + name, data, 'base64');
});
};
mkdirp() function is asynchronous - it creates a directory and returns nothing - this is why you having that leading undefined in the filename.
Save the file in the callback:
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
mkdirp(screenshotPath, function (err) {
if (!err) {
fs.writeFileSync(screenshotPath + name, data, 'base64');
} else {
// handle error
}
});
Or, synchronously create the directory and write to it this way:
var screenshotPath = configuration.readSettings('screenshotDirectory') + fileNameURL + "/";
if (mkdirp.sync(screenshotPath)) {
fs.writeFileSync(screenshotPath + name, data, 'base64');
}

Categories