I am following these instructions to create a basic web scraper that executes in Lambda. I have experience writing selenium code, but not with Node JS. I got the project running in Lambda, but when I tried editing the project locally in order to execute the selenium code I want, It doesn't work. Anything in the exports.handler doesn't get executed when I run node index.js. How would I execute this project locally? Thanks!
This is what I did:
index.js
exports.handler = async (event) => {
console.log('hello world');
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!')
};
return response;
};
package.json
"scripts": {
"locally": "node -e \"console.log(require('./index').handler(require('./event.json')));\""
}
event.json
{
"Records": [
{
"eventVersion": "2.0",
"eventSource": "aws:s3",
"awsRegion": "eu-central-1",
"eventTime": "1970-01-01T00:00:00.000Z",
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId": "AIDAJDPLRKLG7UEXAMPLE"
},
"requestParameters": {
"sourceIPAddress": "127.0.0.1"
},
"responseElements": {
"x-amz-request-id": "C3D13FE58DE4C810",
"x-amz-id-2": "FMyUVURIY8/IgAtTv8xRjskZQpcIZ9KG4V5Wp6S7S/JRWeUWerMUE5JgHvANOjpD"
},
"s3": {
"s3SchemaVersion": "1.0",
"configurationId": "testConfigRule",
"bucket": {
"name": "my-bucket",
"ownerIdentity": {
"principalId": "A3NL1KOZZKExample"
},
"arn": "arn:aws:s3:::my-bucket"
},
"object": {
"key": "HelloWorld.jpg",
"size": 1024,
"eTag": "d41d8cd98f00b204e9800998ecf8427e",
"versionId": "096fKKXTRTtl3on89fVO.nfljtsv6qko"
}
}
}
]
}
Shell
npm run locally
Output
> node -e "console.log(require('./index').handler({}));"
hello world
Promise { { statusCode: 200, body: '"Hello from Lambda!"' } }
You need to call your handler function from another file lets say testHandler.js in order to run via NodeJs.
This will be done like this
//import your handler file or main file of Lambda
let handler = require('./handler');
//Call your exports function with required params
//In AWS lambda these are event, content, and callback
//event and content are JSON object and callback is a function
//In my example i'm using empty JSON
handler.handlerEvent( {}, //event
{}, //content
function(data,ss) { //callback function with two arguments
console.log(data);
});
Now you can use node testHandler.js to test your handler function.
EDIT: Sample Event and content data as requested
Event:
{
"resource": "/API/PATH",
"path": "/API/PATH",
"httpMethod": "POST",
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
"cache-control": "no-cache",
"CloudFront-Forwarded-Proto": "https",
"CloudFront-Is-Desktop-Viewer": "true",
"CloudFront-Is-Mobile-Viewer": "false",
"CloudFront-Is-SmartTV-Viewer": "false",
"CloudFront-Is-Tablet-Viewer": "false",
"CloudFront-Viewer-Country": "IN",
"content-type": "application/json",
"Host": "url.us-east-1.amazonaws.com",
"origin": "chrome-extension://fhbjgbiflinjbdggehcddcbncdddomop",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"Via": "2.0 XXXXXXXXXXXXXX.cloudfront.net (CloudFront)",
"X-Amz-Cf-Id": "XXXXXXXXXX51YYoOl75RKjAWEhCyna-fuQqEBjSL96TMkFX4H0xaZQ==",
"X-Amzn-Trace-Id": "Root=1-XXX03c23-25XXXXXX948c8fba065caab5",
"x-api-key": "SECUREKEY",
"X-Forwarded-For": "XX.XX.XXX.XXX, XX.XXX.XX.XXX",
"X-Forwarded-Port": "443",
"X-Forwarded-Proto": "https"
},
"multiValueHeaders": {
"Accept": [ "*/*" ],
"Accept-Encoding": [ "gzip, deflate, br" ],
"Accept-Language": [ "en-GB,en-US;q=0.9,en;q=0.8" ],
"cache-control": [ "no-cache" ],
"CloudFront-Forwarded-Proto": [ "https" ],
"CloudFront-Is-Desktop-Viewer": [ "true" ],
"CloudFront-Is-Mobile-Viewer": [ "false" ],
"CloudFront-Is-SmartTV-Viewer": [ "false" ],
"CloudFront-Is-Tablet-Viewer": [ "false" ],
"CloudFront-Viewer-Country": [ "IN" ],
"content-type": [ "application/json" ],
"Host": [ "apiurl.us-east-1.amazonaws.com" ],
"origin": [ "chrome-extension://fhbjgbiflinjbdggehcddcbncdddomop" ],
"User-Agent": [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" ],
"Via": [ "2.0 XXXXXXXXXXXXXX.cloudfront.net (CloudFront)" ],
"X-Amz-Cf-Id": [ "XXXXXXXXXhCyna-fuQqEBjSL96TMkFX4H0xaZQ==" ],
"X-Amzn-Trace-Id": [ "Root=1-XXXXXXX67339948c8fba065caab5" ],
"x-api-key": [ "SECUREAPIKEYPROVIDEDBYAWS" ],
"X-Forwarded-For": [ "xx.xx.xx.xxx, xx.xxx.xx.xxx" ],
"X-Forwarded-Port": [ "443" ],
"X-Forwarded-Proto": [ "https" ]
},
"queryStringParameters": null,
"multiValueQueryStringParameters": null,
"pathParameters": null,
"stageVariables": null,
"requestContext": {
"resourceId": "xxxxx",
"resourcePath": "/api/endpoint",
"httpMethod": "POST",
"extendedRequestId": "xxXXxxXXw=",
"requestTime": "29/Nov/2018:19:21:07 +0000",
"path": "/env/api/endpoint",
"accountId": "XXXXXX",
"protocol": "HTTP/1.1",
"stage": "env",
"domainPrefix": "xxxxx",
"requestTimeEpoch": 1543519267874,
"requestId": "xxxxxxx-XXXX-xxxx-86a8-xxxxxa",
"identity": {
"cognitoIdentityPoolId": null,
"cognitoIdentityId": null,
"apiKey": "SECUREAPIKEYPROVIDEDBYAWS",
"cognitoAuthenticationType": null,
"userArn": null,
"apiKeyId": "xxXXXXxxxxxx",
"userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
"accountId": null,
"caller": null,
"sourceIp": "xx.xxx.xxx.xxx",
"accessKey": null,
"cognitoAuthenticationProvider": null,
"user": null
},
"domainName": "url.us-east-1.amazonaws.com",
"apiId": "xxxxx"
},
"body": "{\n \"city\": \"Test 1 City\",\n \"state\": \"NY\",\n \"zipCode\": \"11549\"\n}",
"isBase64Encoded": false
}
Content:
{
"callbackWaitsForEmptyEventLoop": true,
"logGroupName": "/aws/lambda/lambda-name",
"logStreamName": "2018/11/29/[$LATEST]xxxxxxxxxxxb",
"functionName": "lambda-name",
"memoryLimitInMB": "1024",
"functionVersion": "$LATEST",
"invokeid": "xxxxx-xxx-11e8-xxx-xxxxxxxf9",
"awsRequestId": "xxxxxx-xxxxx-11e8-xxxx-xxxxxxxxx",
"invokedFunctionArn": "arn:aws:lambda:us-east-1:xxxxxxxx:function:lambda-name"
}
In your index.js, just defined and exported a handler function, but no one calls it. In the Lambda environment, some AWS code will call this handler with message. In your local environment, you have to call your handler by yourself.
You could also have a look of this doc, it is a way to "simulate" Lambda in local environment.
You can check out lambda-local. It's a little fancier than the accepted answer above. For example, it supports passing environment variables and using JSON files for your payloads.
Perhaps the simplest way to get started, after some testing (with Node 14.17.3)
let handler = require('./index.js');
handler.handler (
{}, // event
{}, // content
(error, result) => {
if (error) console.error(JSON.stringify(error, null, 2));
else console.log(JSON.stringify(result, null, 2));
}
);
Here I am giving a solution for the general case of an synchronous call.
Function signature in your entrypoint index.js:
exports.handler = function(event, context, callback) {
Create a caller file, e.g. testIndex.js
Give it the following contents:
/**
This is a caller for ./index.js
*/
const thatIndex = require('./index');
const EVENT = {
"somekey": {
"somesubkey": "somevalue"
}
}
thatIndex.handler (
// event
EVENT,
// context
{},
// callback function with two arguments
function(err, payload) {
console.log(err);
console.log(payload);
}
);
Go back to your node console (Docker container suggested)
You may use environment variables, if your index.js needs them: define each with export MYVAR1=myvalue1
Run your caller: node testIndex.js
The console should output the payload, provided your main file (entrypoint) promise call is chained-up with: (example)
.then(() => callback(null, {
statusCode: 301,
headers: {'location': process.env.URL + '/' + somekey },
body: '',
})
)
.catch(err => callback(err))
Tip: you may need to clean up the cache with node clean cache if you notice that it's the previous code that's called.
Other Tip: in the AWS environment, you don't need to zip up the node_modules/aws-sdk folder, as it will be part of the standard execution context. But on your test machine you will need to. So, in development mode, simple run npm install, not npm install --omit=dev
So, before zipping the folder that contains your function, just remove node_modules folder recursively, and rebuild the dependencies with npm install --ommit=dev
Example of zipping the folder:
zip -r ..\resizeImage.zip .
This is what your package.json file could contain:
"dependencies": {
"sharp": "^0.27.2"
},
"devDependencies": {
"aws-sdk": "^2.1195.0"
}
If you want just to execute it locally you can use the official sam cli tool
https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-cli-command-reference-sam-local-invoke.html
If you use VSCode you can also check out this extension:
https://marketplace.visualstudio.com/items?itemName=bogdan-onu.invoke
IMO. The best way to test a lambda is to actually just test it! What does that mean? Will simply use some testing library (like jest for example) and simply create a test on your handler function. Mock anything you need and provide some data you expect that will come into the lambda (for the event and context if needed). And that's it. You have some tests written and a quick way to test your lambda at the same time.
Related
I've got problem in proxy authentication using chrome extension (due to problems with v3 my code is based on manifest v2). The connection works using the following curl command:
curl -v -x https://11.111.11.111:222 https://www.google.com/ --proxy-header "proxy-authorization: Basic abc" --proxy-insecure
And I tried to implement it in extension. Here's what I have in background.js file:
chrome.runtime.onStartup.addListener(() => {
const config = {
mode: 'fixed_servers',
rules: {
singleProxy: {
host: '11.111.11.111',
port: 222,
scheme: 'https',
},
},
}
chrome.proxy.settings.set({ value: config, scope: 'regular' })
})
chrome.webRequest.onBeforeSendHeaders.addListener(
(details) => {
const requestHeaders = details.requestHeaders || []
requestHeaders.push({
name: 'proxy-authorization',
value: 'Basic abc',
})
return { requestHeaders }
},
{ urls: ['<all_urls>'] },
['blocking', 'requestHeaders', 'extraHeaders']
)
chrome.webRequest.onSendHeaders.addListener(
(details) => {
console.log('sending', details)
return details
},
{ urls: ['<all_urls>'] },
['requestHeaders', 'extraHeaders']
)
I have the following permissions in manifest.json:
"permissions": [
"cookies",
"storage",
"tabs",
"activeTab",
"proxy",
"webRequest",
"webRequestBlocking",
"<all_urls>"
]
And these are the headers that are printed in onSendHeaders function:
You see that proxy-authorization header is present. But I get ERR_PROXY_CERTIFICATE_INVALID while trying to browse any page. Proxy headers should be set in a different way? Because I use --proxy-header flag in the curl command, not --header.
PS. I tried using the method that is mentioned many times:
chrome.webRequest.onAuthRequired.addListener(function(details, callbackFn) {
callbackFn({
authCredentials: { username: username, password: password }
});
},{urls: ["<all_urls>"]},['asyncBlocking']);
too, but this function is never called (I put console.log inside).
Have you solved this problem?
If not try this:
chrome.webRequest.onAuthRequired.addListener(function(details, callbackFn) {
callbackFn({
authCredentials: { username: username, password: password }
});
},{urls: ["<all_urls>"]},['blocking']);
Main change - AsyncBlocking => Blocking
I have a custom subdomain connected to Firebase Hosting - api.example.com
Additionally I have a function in Firebase Functions
exports.test = functions
.region('europe-west3')
.https.onRequest((request, response) => {
const expiresIn = 60 * 60 * 24 * 5 * 1000;
const origin = request.headers.origin;
const options = { maxAge: expiresIn, httpOnly: true, secure: true, sameSite: 'strict', };
response.cookie('__session', "123456", options);
response.set('Access-Control-Allow-Origin', origin);
response.set('Access-Control-Allow-Credentials', true);
response.end(JSON.stringify({ status: 'success' }));
})
my Firebase.json looks like the following
{
"functions": {
"source": "functions"
},
"hosting": {
"public": "public",
"ignore": [
"firebase.json",
"**/.*",
"**/node_modules/**"
],
"rewrites": [{
"source": "/test",
"function": "test"
}]
}
}
Now when I'm trying to make a POST request to https://api.example.com/test
I get a error
Your client does not have permission to get URL /test/test from this server.
which makes sense as the URL says /test/test/
What am I doing wrong here? Any suggestions? Thanks.
Firebase Hosting does not support regions other than us-central1 for Cloud Functions. See the documentation:
Important: Firebase Hosting supports Cloud Functions in us-central1 only.
See also this issue on GitHub: https://github.com/firebase/firebase-tools/issues/842
I am trying to create the Profile model in loopback it is showing the error.
error details
Unhandled error for request POST /api/Users: Error: Cannot call Profile.create(). The create method has not been setup. The PersistedModel has not been correctly attached to a DataSource!
'use strict';
var loopback = require('loopback');
var boot = require('loopback-boot');
var app = module.exports = loopback();
app.start = function() {
// start the web server
return app.listen(function() {
app.emit('started');
var baseUrl = app.get('url').replace(/\/$/, '');
console.log('Web server listening at: %s', baseUrl);
if (app.get('loopback-component-explorer')) {
var explorerPath = app.get('loopback-component-explorer').mountPath;
console.log('Browse your REST API at %s%s', baseUrl, explorerPath);
}
});
};
// Bootstrap the application, configure models, datasources and middleware.
// Sub-apps like REST API are mounted via boot scripts.
boot(app, __dirname, function(err) {
if (err) throw err;
// start the server if `$ node server.js`
if (require.main === module)
app.start();
});
console.log(Object.keys(app.models));
app.models.User.afterRemote('create',(ctx,user,next)=>{
console.log("The new User is ",user);
app.models.Profile.create({
first_name :user.username,
created_at :new Date(),
userId: user.id
}, (err,result)=>{
if(!err && result)
{
console.log("Created new profile !", result);
}
else{
console.log("There is an error ",err);
}
next();
});
});
Prfile.JSON file
{
"name": "Profile",
"base": "PersistedModel",
"idInjection": true,
"options": {
"validateUpsert": true
},
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
},
"birth_date":{
"type":"date"
},
"created_at": {
"type": "date"
},
"age": {
"type": "number"
},
"history":{
"type":["object"]
}
},
"validations": [],
"relations": {},
"acls": [],
"methods": {}
}
**This file is the profile.js file **
'use strict';
module.exports = function(Profile) {
};
Perhaps you still need to bind your model to the data source in the model-config.json file, like that:
datasources.json
"rdb": {
"host": "localhost",
"port": 3306,
"database": "asat",
"password": "12345",
"name": "rdb",
"user": "admin",
"connector": "mysql"
}
...
model-config.json
"Profile": {
"dataSource": "rdb",
"public": true
}
...
The reason can be any of the follwoing
Forget to add the model name in the model-config.json file
Sometime when the loopback server start the model didnt get appended to the server object properly. If the model is not appended properly we can not use the model using the app object and one server restart will solve the issue.
Datasource is not correctly mentioned in model-config file
I am following loopback environment specific configuration to refer environment specific config, data sources and middleware json files.
When I push my app to a staging server, NODE_ENV environment variable gets set to staging and based on this variable config.staging.json and datasources.staging.json files are referred but NOT middleware.staging.json.
Here is the complete content of middleware.json file:
{
"initial:before": {
"loopback#favicon": {}
},
"initial": {
"compression": {},
"cors": {
"params": {
"origin": true,
"credentials": true,
"maxAge": 86400
}
}
},
"session": {},
"auth": {},
"parse": {},
"routes": {},
"files": {
"loopback#static": [
{
"params": "$!../client/WebApp/www"
}
]
},
"final": {
"loopback#urlNotFound": {}
},
"final:after": {
"errorhandler": {}
}
}
and I would like to refer middleware.staging.json file whose complete content is:
{
"files": {
"loopback#static": [
{
"params": "$!../client/WebApp/dist"
}
]
}
}
I am trying to change the target folder from www to dist. It seems middleware.staging.json file is not referred and middleware.json file is always referred irrespective of NODE_ENV variable.
Am I creating environment specific middleware file correctly? Any alternates to achieve this?
I am using Grant for OAuth2 authentication with google. I supplied all the parameters in the config.json :
{
"server": {
"protocol": "https",
"host": "thooslo-com-shaunakde.c9.io"
},
"google":{
"authorize_url": "https://accounts.google.com/o/oauth2/auth",
"access_url": "https://accounts.google.com/o/oauth2/token",
"oauth": 2,
"custom_parameters": ["access_type"],
"scope_delimiter":" ",
"scope":["https://www.googleapis.com/auth/youtube","https://www.googleapis.com/auth/drive"],
"client_id":"39109025743-veaeooi4v9ooirabeseujn8u2ohjbqf7.apps.googleusercontent.com",
"client_secret":"DO8ozwoFqtP654jzi-wPQF10",
"callback": "/users"
}
}
But it still refuses to send all the parameters. I get a "client_id" not sent error.
I modified the library to print out the URL and this is indeed the case:
Starting child process with 'node ./bin/www'
https://accounts.google.com/o/oauth2/auth?response_type=code&redirect_uri=https%3A%2F%2Fthooslo-com-shaunakde.c9.io%2Fconnect%2Fgoogle%2Fcallback&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fyoutube%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive
GET /connect/google 302 26.492 ms - 574
I did manage to get the first step to work once, by some permutation, but then it failed on client_secret not found.
Is there something I am doing wrong? The library seems to be non buggy.
It's sufficient to have this configuration
{
"server": {
"protocol": "https",
"host": "thooslo-com-shaunakde.c9.io"
},
"google":{
"key":"39109025743-veaeooi4v9ooirabeseujn8u2ohjbqf7.apps.googleusercontent.com",
"secret":"DO8ozwoFqtP654jzi-wPQF10",
"scope":[
"https://www.googleapis.com/auth/youtube",
"https://www.googleapis.com/auth/drive"
],
"callback": "/users"
}
}