We Keep Coding

Can a Twilio Function call a Redis instance in the cloud?

I'm trying to call Redis from a Twilio Function (serverless) and I don't see incoming connections in my Redis log.
Is this setup viable?
Sample code follows:
const Redis = require('ioredis');
const fs = require('fs');
exports.handler = function (context, event, callback) {
const config = Runtime.getAssets()['config.json'].open();
let redisClientConfig = JSON.parse(config).redisConfig;
let contactCacheTime = JSON.parse(config).contactCacheTime;
if (!redisClientConfig) {
throw new Error('Redis config not set.');
}
const redisClient = new Redis(redisClientConfig);
redisClient.on('error', (err) => {
console.error(`Cannot connect to redis, reason: ${(err.message || err)}`);
});
redisClient.getex('mhn-twilio-bot-contact:'.concat(event.contactKey), 'EX', contactCacheTime)
.then((res) => {
if (!res) {
redisClient.setex('mhn-twilio-bot-contact:'.concat(event.contactKey), contactCacheTime, '<CACHED-VALUE>');
}
callback(null, { cached: res ? true : false });
})
.catch((err) => {
callback(null, { cached: false });
});
};

why the result of uploader is error but never goes in .catch in requestPromise

this is my test file for upload and i explain it step by step:
I wrote a test to upload the file. the uploader method written with busboy module and it working true
but i have problem in test.
when result of uploader is error, this error never returned in .catch and go in .then.
more explain in code:
const http = require('http');
// const request = require('request');
const rp = require('request-promise');
const fs = require('fs');
const assert = require('chai').assert;
const port = process.env.PORT || 80;
const Q = require('q');
let server;
const options = {
method: 'POST',
uri: 'http://127.0.0.1/upload',
formData: {
name: 'test',
file: {
value: fs.createReadStream('./test/test.jpg'),
options: {
filename: 'test.jpg',
contentType: 'image/jpg'
}
}
},
headers: {
'Connection': 'Keep-Alive',
'content-type': 'multipart/form-data' // Is set automatically
},
json: true,
};
function startServer(port, cb) {
server = http.createServer(function (req, res) {
if (req.method === 'POST') {
if (req.url === '/upload') {
serveRequest(req, res);
}
}
});
server.listen(port, () => {
cb(function stopServer(done) {
setTimeout(function () {
server.close();
done();
}, 20);
});
console.log(`listening on port ${port} ...`);
});
}
function serveRequest(request, response) {
if (request.headers.hasOwnProperty('content-type')
&& request.headers['content-type'].split(';')[0] === 'multipart/form-data') {
serveUpload(request, response);
}
}
function serveUpload(request, response) {
uploader.upload(request, function (error, res) {
if (error) {
response.end();
}
else {
response.write(JSON.stringify(res));
response.end();
}
});
}
// -----------------------
describe('upload', function () {
let stopServer = null;
before('start server', function (done) {
startServer(port, function (stop) {
stopServer = stop;
done();
});
});
it('upload a file - options is true', function (done) {
rp(options)
.then(function (r) {
console.log(r);
})
.catch(function (error) {
console.log(error);
});
});
after('stop server', function (done) {
stopServer(done);
});
});
I make a request to the uploader and the result of my request is returned in the serveUpload() method. The result of serveUpload() is error and error is object like this :
error =
meta: {
code: '',
sourceType: 'module',
sourceName: '',
version: '2.0.4'
},
data: {
message: {
en: 'uploaded data size is out of limit'
}
}
}
this error must returned .catch(e) in the rp(options), but in fact it must go to .then(r) in rp(options)
log r in .then is undefined.
rp(options)
.then(function (r) {
console.log(r); // error always come here But in fact it must go to catch and r is undefined
})
.catch(function (error) {
console.log(error);
});
I don't understand why this is happening, I would be very grateful if anyone could help me.

Sending multiple files to google bucket with multer - why are my files empty?

I previously had a single file upload set up and working properly. Now I need to make it handle multiple files.
Here is my code right now:
const multer = require('multer')
const { Storage } = require('#google-cloud/storage')
const storage = new Storage()
const m = multer({ storage: multer.memoryStorage() })
module.exports = app => {
app.use('/', router)
router.post(
'/reader-:shortId/file-upload',
passport.authenticate('jwt', { session: false }),
m.array('files'),
async function (req, res) {
const bucketName = req.params.shortId.toLowerCase()
await storage.createBucket(bucketName)
bucket = storage.bucket(bucketName)
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype }
}).on('finish', async response => {
await blob.makePublic()
resolve(response)
}).on('error', err => {
reject('upload error: ', err)
}).end()
})
promises.push(newPromise)
})
Promise.all(promises).then((response) => {
// the response I get here is [undefined, undefined]
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
})
}
req.files does give me an array of files, with a buffer and a size that makes sense.
The promises all resolve.
But once I check the files in the google bucket, they have the right name but don't have any content (and size of 0)
As I said before, it was working when I was doing it with one file (using m.single('file')
I don't want to use the bucket as the destination with multer setup because I also have to change the file name before uploading to google bucket.
edit: this is the code example given by google cloud documentations for single file uploads (https://cloud.google.com/nodejs/getting-started/using-cloud-storage):
function sendUploadToGCS (req, res, next) {
if (!req.file) {
return next();
}
const gcsname = Date.now() + req.file.originalname;
const file = bucket.file(gcsname);
const stream = file.createWriteStream({
metadata: {
contentType: req.file.mimetype
},
resumable: false
});
stream.on('error', (err) => {
req.file.cloudStorageError = err;
next(err);
});
stream.on('finish', () => {
req.file.cloudStorageObject = gcsname;
file.makePublic().then(() => {
req.file.cloudStoragePublicUrl = getPublicUrl(gcsname);
next();
});
});
stream.end(req.file.buffer);
}
I originally had something like that working, but I just don't understand where it is getting the file buffer data from. That is probably where things are different with multiple files.

I know its too late, but someone might looking an answer for uploading multiple files on Google Cloud Storage.
Dependencies:
Express
Google Cloud Library
Multer
Body Parser
This is the controller code.
exports.post_image_upload = async (req, res) => {
/** Check if file exist */
if (!req.files) {
res.status(400).send('No file uploaded.');
return;
}
let PublicUrls = []
req.files.forEach((file) => {
const blob = bucket.file(file.fieldname + '-' + Date.now() + path.extname(file.originalname))
const blobStream = blob.createWriteStream({
metadata: { contentType: file.mimetype }
})
blobStream.on('finish', ()=> {
blob.makePublic()
})
blobStream.on('error', err => {
//Put your error message here
})
blobStream.end(file.buffer)
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`
PublicUrls.push(Url)
})
res.send(PublicUrls)
}
Good Luck

Ok, turns out I had to change
.end()
to
.end(file.buffer)

Marie Pelletier, I think your approach is 100% right. I modified a little your code trying to avoid the async response:
let promises = []
req.files.forEach((file) => {
const blob = bucket.file(file.originalname)
const newPromise = new Promise((resolve, reject) => {
blob.createWriteStream({
metadata: { contentType: file.mimetype },
resumable: false //Good for small files
}).on('finish', () => {
const Url = `https://storage.googleapis.com/${bucket.name}/${blob.name}`;
resolve({ name: file.originalname, url: Url });
}).on('error', err => {
reject('upload error: ', err);
}).end(file.buffer);
})
promises.push(newPromise);
})
Promise.all(promises).then((response) => {
res.status(200).send(response)
}).catch((err) => {
res.status(400).send(err.message)
});
This way, I didn't get 'undefined' anymore.

Dynamically accessing function arguments

Hi have the following simple ExpressJS application, where the routes are dynamically created based on a configuration. I am having a hard time trying to pass in a bunch of parameters to the handler so that the values are returned in the respective controller.
const express = require('express');
module.exports = class App {
get routes() {
return [
{
path: '/',
verb: 'get',
method: 'home',
params: ['req.query.ref', 'req.query.country'],
},
];
}
constructor() {
this.app = express();
this.register();
}
register() {
const { routes } = this;
routes.forEach((route) => {
const {
path, verb, method, params,
} = route;
// if you replace the params with [req.query.ref, req.query.country] it will work as expected
this.app[verb](path, this.handler(this[method].bind(this), (req, res, next) => params));
});
}
handler(promise, params) {
return async (req, res, next) => {
const bound = params ? params(req, res, next) : [];
console.log(bound);
try {
const result = await promise(...bound);
res.json(result);
} catch (err) {
throw err;
}
};
}
home(payload) {
console.log(payload);
return Promise.resolve({ status: 'OK' });
}
};

Most of your issues stem from the structure of your route definition. It would make more sense to create direct references to the things you want to use, not noting function references etc down as strings.
get routes() {
return [{
path: '/',
method: this.get,
endpoint: this.home,
paramMap: req => [req.query.ref, req.query.country],
}];
}
Once you make the appropriate changes elsewhere, you no longer have the original problem you described.

Maybe you can take a look to the arguments object. All functions have this object and it contains an array with all arguments received in the function. I think it could be what you are looking for.
JavaScript functions have a built-in object called the arguments object.
The argument object contains an array of the arguments used when the function was called (invoked).
This way you can simply use a function to find (for instance) the highest value in a list of numbers:
This is an example how it works:
x = findMax(1, 123, 500, 115, 44, 88);
function findMax() {
var i;
var max = -Infinity;
for (i = 0; i < arguments.length; i++) {
if (arguments[i] > max) {
max = arguments[i];
}
}
return max;
}
More info:
https://www.w3schools.com/js/js_function_parameters.asp

well you can build the query if with split if your params has the same format
--update--
this solution is based on req or res with x params for each
const express = require('express');
module.exports = class App {
get routes() {
return [
{
path: '/',
verb: 'get',
method: 'home',
params: ['req.query.ref', 'req.query.country'], //changed
},
];
}
constructor() {
this.app = express();
this.register();
}
register() {
const { routes } = this;
routes.forEach((route) => {
let {
path, verb, method, params,
} = route;
this.app[verb](path, this.handler(this[method].bind(this), (req, res, next) => this. paramsStringToArrayValues(req, res,params))
}));
});
}
paramsStringToArrayValues(req, res,params){
return params.map(param => {
let paramArr = param.split('.');
let obj = paramArr.shift() === 'req'? req : res
paramArr.forEach(key =>{
obj = obj[key]
})
return obj
})
}
handler(promise, params) {
return async (req, res, next) => {
const bound = params ? params(req, res, next) : [];
console.log(bound);
try {
const result = await promise(...bound);
res.json(result);
} catch (err) {
throw err;
}
};
}
home(payload) {
console.log(payload);
return Promise.resolve({ status: 'OK' });
}
};

Testing firebase HTTPS call with CORS

I am trying to setup a local testing environment for my firebase cloud functions. However I run into problems when trying to do a fake call to one of my HTTP functions.
The reason for my error seems to be that I am using CORS (npm). When I remove cors and run the function "test" seen below with just a response.status(200) everything works. But when wrapping with cors(req,res) my test fails with TypeError: Cannot read property 'origin' of undefined.
What am I doing wrong here?
In index.js -->
exports.test = functions.https.onRequest((request, response) => {
cors(request, response, () => {
response.status(200);
response.send("test ok");
})
In my test.js
describe('Cloud Functions', () => {
// [START stubConfig]
var myFunctions, configStub, adminInitStub, functions, admin, cors;
before(() => {
// Since index.js makes calls to functions.config and admin.initializeApp at the top of the file,
// we need to stub both of these functions before requiring index.js. This is because the
// functions will be executed as a part of the require process.
// Here we stub admin.initializeApp to be a dummy function that doesn't do anything.
admin = require('firebase-admin');
cors = require('cors')({
origin: true
});
adminInitStub = sinon.stub(admin, 'initializeApp');
// Next we stub functions.config(). Normally config values are loaded from Cloud Runtime Config;
// here we'll just provide some fake values for firebase.databaseURL and firebase.storageBucket
// so that an error is not thrown during admin.initializeApp's parameter check
functions = require('firebase-functions');
configStub = sinon.stub(functions, 'config').returns({
firebase: {
databaseURL: 'https://not-a-project.firebaseio.com',
storageBucket: 'not-a-project.appspot.com',
}
// You can stub any other config values needed by your functions here, for example:
// foo: 'bar'
});
// Now we can require index.js and save the exports inside a namespace called myFunctions.
// This includes our cloud functions, which can now be accessed at myFunctions.makeUppercase
// and myFunctions.addMessage
myFunctions = require('../index');
});
after(() => {
// Restoring our stubs to the original methods.
configStub.restore();
adminInitStub.restore();
});
// [END stubConfig]
describe('test', () => {
it('should return status code 200', (done) => {
// [START invokeHTTPS]
// A fake request object, with req.query.text set to 'input'
const req = {};
// A fake response object, with a stubbed redirect function which asserts that it is called
// with parameters 303, 'new_ref'.
const res = {
status: (status) => {
assert.equal(status, 200);
done();
}
};
// Invoke addMessage with our fake request and response objects. This will cause the
// assertions in the response object to be evaluated.
myFunctions.test(req, res);
// [END invokeHTTPS]
})
})
})

Try this:
Instead of using const req = {}; use:
const req = {
headers: { origin: true },
};

Here's how I got around the cors errors, with sinon
const res = {};
Object.assign(res, {
status: sinon.stub().returns(res),
end: sinon.stub().returns(res),
json: sinon.stub().returns(res),
setHeader: sinon.stub(),
getHeader: sinon.stub(),
});
beforeEach(() => {
Object.values(res).forEach(stub => stub.resetHistory());
});
Then, in your test, you can test your responses:
cloudFunctions.testFunction({ query: { text: 'foo' } }, res);
response = res.json.lastCall.args[0];

Finaly the unit test with firebase function and cors is working
You have to setHeader and GetHeader in the response
setHeader: (key, value) => {},
getHeader: (value) => {},
The test
const test = require('firebase-functions-test')()
let myFunctions
beforeAll(() => {
myFunctions = require('../src/index')
})
afterAll(() => {
test.cleanup()
})
it('get', async () => {
let request = { headers: { origins: true } }
const response = {
setHeader: (key, value) => {
},
getHeader: (value) => {
},
status: (code) => {
expect(code).toBe(200)
return { send: (body) => {
expect(body).toBe('ok')
}
}
}
}
myFunctions.addLink(request, response)
})
The Index
const functions = require('firebase-functions')
const cors = require('cors')({
origin: true
})
exports.addLink = functions.https.onRequest((req, res) => {
return cors(req, res, () => {
res.status(200).send('ok')
})
})

add headers: { origin: '*' }, to your request and setHeader () {} to response
const req = {
{ origin: '*' }
};
const res = {
status: (status) => {
assert.equal(status, 200);
done();
}
};