We Keep Coding

Rate exceeded while running stop services script Fargate

Really need help with this AWS Fargate stopping script, Im receiving an error called "Rate exceeded" "ThrottilingException", is there a way to add that my script detects the error and if it sees the error, it will pause for a little bit and continue where it was left off?
var AWS = require("aws-sdk");
AWS.config.update({region: "eu-central-1"});
var s3 = new AWS.S3({apiVersion: '2006-03-01'});
var ecs = new AWS.ECS({apiVersion: '2014-11-13'});
async function getServiceArns(token) {
var params = {
cluster: 'my-cluster',
launchType: "FARGATE",
maxResults: 100,
nextToken: '',
schedulingStrategy: "REPLICA"
};
return new Promise(function(resolve) {
var services = []
if (token) {
params.nextToken = token
}
ecs.listServices(params, async function(err, data) {
if(data.nextToken) {
services.push(...(await getServiceArns(data.nextToken)))
}
services.push(...data.serviceArns)
resolve(services)
});
})
}
async function main(){
var arns = await getServiceArns()
var services = await Promise.all(arns.map(function(arn) {
return new Promise(function(resolve) {
ecs.describeServices({cluster: 'my-cluster', services:[arn]}, function(err, data) {
resolve({arn, count: data.services[0].desiredCount})
});
})
}))
var filtered = services.filter(item => item.count > 0)
var base64data = Buffer.from(JSON.stringify(filtered), 'binary');
await s3.putObject({
Bucket: 'my-bucket',
Key: 'services.txt',
Body: base64data,
}).promise();
await Promise.all(filtered.map(function(service) {
var params = {
desiredCount: 0,
cluster: "my-cluster",
service: service.arn,
};
console.log(service.arn)
// return ecs.updateService(params).promise()
}))
console.log('successfully shutdown all services')
}
exports.handler = main

NodeJS/knex How can I handle each item received when querying from the DB (records over 1 billion)

Let's say there are 1 billion+ records in the database, I need to send each record for verification.
If I use option 1, then the "memory leaks" (when using 1 million records, memory usage about 5Gb).
If I use option 2, then as expected I get an error about exceeding a 32 bit value (Timeout duration was set to 1. TimeoutOverflowWarning: 2164644000 does not fit into a 32-bit signed integer.)
*In addition, the request to check takes ~ 1-2 seconds
const url = require('url');
const http = require('http');
const knexOptions = {
client: 'mysql',
connection: {
host: '127.0.0.1',
user: 'root',
password: 'password',
database: 'test0000'
}
}
const knex = require('knex')(knexOptions);
knex.select('*').from('addresses')
.then(function(result) {
/*
//option 1
result.forEach(function(record) {
//console.log(record);
makeRequest(record);
});
*/
//option 2
result.forEach(function(record, index) {
setTimeout(function() {
makeRequest(record);
}, 3000 * (index + 1)); // or just index, depends on your needs
});
});
function makeRequest(record) {
let stringifyString = JSON.stringify(record);
let parsedJSON = JSON.parse(stringifyString);
//console.log(parsedJSON.address);
let _url = 'http://localhost/RestAPI.php?address=' + parsedJSON.address;
let urlHost = url.parse(_url).host;
let urlPath = url.parse(_url).path;
var options = {
host: urlHost,
path: urlPath
};
callback = function(response) {
let str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
//console.log(str);
let parsedJSON = JSON.parse(str);
console.log(parsedJSON.valid);
if (parsedJSON.valid) {
//true received
knex('addresses')
.update({valid: true})
.where('address', parsedJSON.address)
.then(function(result) {
//console.log(result);
})
.catch((err) => {
console.log(err);
});
}
});
}
let req = http.request(options, callback);
req.end();
}
Can the community suggest a more elegant method/solution to my problem?
I appreciate your help!

Node js promise chaining issue

I am trying to connect to Salesforce using node js / jsforce library and use promises. Unfortunately one of the methods is executing prior to getting connection.
i have method A : makeconnection which returns the connection
i have method B : which loads data from Salesforce based on the connection reference from method A
I have method C : which gets dependencies from Salesforce based on connection from method A
I would like the following order to be executed A ==> B ==> C
Unfortunately C seems to run first followed by A and B so the connection is null and it fails
roughly this is the code
let jsforce = require("jsforce");
const sfdcSoup = require("sfdc-soup");
const fs = require("fs");
let _ = require("lodash");
let trgarr = [];
let clsarr = [];
let entityarr = [];
function makeConnection() {
return new Promise((resolve,reject) => {
const conn = new jsforce.Connection({
loginUrl: "https://test.salesforce.com",
instanceUrl: "salesforce.com",
serverUrl: "xxx",
version: "50.0"
});
conn.login(username, password, function (err, userInfo) {
if (err) {
return console.error(err);
}
// console.log(conn.accessToken);
//console.log(conn.instanceUrl);
//console.log("User ID: " + userInfo.id);
//console.log("Org ID: " + userInfo.organizationId);
console.log("logged in");
});
resolve(conn);
});
}
function loadClasses(conn) {
return new Promise((resolve,reject) => {
const querystr =
"select apiVersion,name,body from apexClass where NamespacePrefix = null";
let query = conn
.query(querystr)
.on("record", function (rec) {
clsarr.push(rec);
})
.on("end", function () {
console.log("number of class is " + clsarr.length);
console.log("loaded all classes");
});
resolve(conn,clsarr);
});
}
async function getDependencies(conn) {
return new Promise((resolve,reject) => {
let entryPoint = {
name: "xxx",
type: "CustomField",
id: yyy
};
let connection = {
token: conn.accessToken,
url: "abc.com",
apiVersion: "50.0"
};
let usageApi = sfdcSoup.usageApi(connection, entryPoint);
usageApi.getUsage().then((response) => {
console.log(response.stats);
console.log(response.csv);
});
});
}
async function run() {
makeConnection().then(conn => loadClasses(conn)).then(conn=>getDependencies(conn));
}
run();
I keep getting an error that says UnhandledPromiseRejectionWarning: Error: Access token and URL are required on the connection object
The reason is connection needs to be obtained from method A and sent to Method C , which is not happening. Can you please guide where i might be wrong?
Also why is method C getting executed before A and B. why does my promise chaining not work as promised?
I am running the code in Vscode and using Node 14

Your 2 method have minor correction first method makeConnection, the resolve should be inside login after console.log("logged in")
And second loadClasses, the resolve should be inside 'end' event. Please check below 2 method.
function makeConnection() {
return new Promise((resolve,reject) => {
const conn = new jsforce.Connection({
loginUrl: "https://test.salesforce.com",
instanceUrl: "salesforce.com",
serverUrl: "xxx",
version: "50.0"
});
conn.login(username, password, function (err, userInfo) {
if (err) {
return console.error(err);
}
// console.log(conn.accessToken);
//console.log(conn.instanceUrl);
//console.log("User ID: " + userInfo.id);
//console.log("Org ID: " + userInfo.organizationId);
console.log("logged in");
resolve(conn);
});
});
}
function loadClasses(conn) {
return new Promise((resolve,reject) => {
const querystr =
"select apiVersion,name,body from apexClass where NamespacePrefix = null";
let query = conn
.query(querystr)
.on("record", function (rec) {
clsarr.push(rec);
})
.on("end", function () {
console.log("number of class is " + clsarr.length);
console.log("loaded all classes");
resolve(conn,clsarr);
});
});
}

you should use promise series if methods are depending on each other if methods do not depend then you should use a promise parallel.
READ MORE ABOUT PROMISE SERIES AND PARALLEL.

Connection resets after 60 seconds in node.js upload application

I've written an application in node.js consisting of a server and a client for storing/uploading files.
For reproduction purposes, here's a proof of concept using a null write stream in the server and a random read stream in the client.
Using node.js 12.19.0 on Ubuntu 18.04. The client depends on node-fetch v2.6.1.
The issue I have is after 60 seconds the connection is reset and haven't found a way to make this work.
Any ideas are appreciated.
Thank you.
testServer.js
// -- DevNull Start --
var util = require('util')
, stream = require('stream')
, Writable = stream.Writable
, setImmediate = setImmediate || function (fn) { setTimeout(fn, 0) }
;
util.inherits(DevNull, Writable);
function DevNull (opts) {
if (!(this instanceof DevNull)) return new DevNull(opts);
opts = opts || {};
Writable.call(this, opts);
}
DevNull.prototype._write = function (chunk, encoding, cb) {
setImmediate(cb);
}
// -- DevNull End --
const http = require('http');
const server = http.createServer();
server.on('request', async (req, res) => {
try {
req.socket.on('end', function() {
console.log('SOCKET END: other end of the socket sends a FIN packet');
});
req.socket.on('timeout', function() {
console.log('SOCKET TIMEOUT');
});
req.socket.on('error', function(error) {
console.log('SOCKET ERROR: ' + JSON.stringify(error));
});
req.socket.on('close', function(had_error) {
console.log('SOCKET CLOSED. IT WAS ERROR: ' + had_error);
});
const writeStream = DevNull();
const promise = new Promise((resolve, reject) => {
req.on('end', resolve);
req.on('error', reject);
});
req.pipe(writeStream);
await promise;
res.writeHead(200);
res.end('OK');
} catch (err) {
res.writeHead(500);
res.end(err.message);
}
});
server.listen(8081)
.on('listening', () => { console.log('Listening on port', server.address().port); });
testClient.js
// -- RandomStream Start --
var crypto = require('crypto');
var stream = require('stream');
var util = require('util');
var Readable = stream.Readable;
function RandomStream(length, options) {
// allow calling with or without new
if (!(this instanceof RandomStream)) {
return new RandomStream(length, options);
}
// init Readable
Readable.call(this, options);
// save the length to generate
this.lenToGenerate = length;
}
util.inherits(RandomStream, Readable);
RandomStream.prototype._read = function (size) {
if (!size) size = 1024; // default size
var ready = true;
while (ready) { // only cont while push returns true
if (size > this.lenToGenerate) { // only this left
size = this.lenToGenerate;
}
if (size) {
ready = this.push(crypto.randomBytes(size));
this.lenToGenerate -= size;
}
// when done, push null and exit loop
if (!this.lenToGenerate) {
this.push(null);
ready = false;
}
}
};
// -- RandomStream End --
const fetch = require('node-fetch');
const runSuccess = async () => { // Runs in ~35 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(256e6) // new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6)
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
// runSuccess().then(() => process.exit(0));
runFail().then(() => process.exit(0));

I tried (unsuccessfully) to reproduce what you are seeing based on your code example. Neither the success call is completing in ~35 seconds nor is the error being thrown in 60 seconds.
However, that being said, I think what is happening here is that your client is terminating the request.
You can increase the timeout by adding a httpAgent to the fetch PUT call. You can then set a timeout in the httpAgent.
const http = require('http');
...
const runFail = async () => { // Fails after 60 seconds
const t = Date.now();
try {
const resp = await fetch('http://localhost:8081/test', {
method: 'PUT',
body: new RandomStream(1024e6),
agent: new http.Agent({ keepAlive: true, timeout: 300000 })
});
const data = await resp.text();
console.log(Date.now() - t, data);
} catch (err) {
console.warn(Date.now() - t, err);
}
};
See the fetch docs for adding a custom http(s) agent here
See options for creating http(s) agent here

This turned out to be a bug in node.js
Discussion here: https://github.com/nodejs/node/issues/35661

Close Event Triggers Before Data Events on File Stream

I've got a script that adds JSON data from a file to a DynamoDB table. The script uses the "fs" module to open a read stream to the json file and retrieve the data line by line. As the data is returned, it's inserted into a DynamoDB table. When the operation ends, an execution summary is given with number of records processed, successfully inserted, and unsuccessfully inserted. The problem is the summary executes before the file has completely processed. As result the numbers are wrong.
The script...
ddb_table_has_records(table_name, (err, dat) => {
if (dat.Count === 0 || force) {
const transformStream = JSONStream.parse("*");
const inputStream = fs.createReadStream(import_file);
let record_position = 0;
let count_imported_successful = 0;
let count_imported_fail = 0;
inputStream.pipe(transformStream).on("data", (Item) => {
const params = {
TableName: table_name,
Item
}
ddb_client.put(params, (err, data) => {
++record_position;
if (err) {
console.error("Unable to add mapping for record " + record_position + ", error = " + err);
++count_imported_fail;
} else {
console.log("PutItem succeeded " + record_position);
++count_imported_successful;
}
});
}).on("close", () => {
console.log("=".repeat(70));
console.log(`'Completed: ${import_file}' has been loaded into '${table_name}'.`);
console.log(` Record Count: ${record_position}`);
console.log(` Imported Record Count: ${count_imported_successful}`);
console.log(` Rejected Record Count: ${count_imported_fail}`);
});
} else {
console.log("=".repeat(70));
console.log(`Completed: Skipping import of '${import_file}' into '${table_name}'.`);
};
});
When this runs, it looks like the following
PS C:\> node --max-old-space-size=8192 .\try.js 'foo' 'us-west-2' 'development' '.\data.json' true
Target Profile: development
Target Region: us-west-2
Target Table: foo
Source File: .\data.json
Force Import: true
Confirming Table's State...
======================================================================
'Completed: .\data.json' has been loaded into 'foo'.
Record Count: 0
Imported Record Count: 0
Rejected Record Count: 0
PutItem succeeded 1
PutItem succeeded 2
PutItem succeeded 3
PutItem succeeded 4
...
The portion of the code that gets the record counts runs before the inserts completes so the records imported and rejected numbers are always wrong. It looks like the file stream closes while inserts are occurring. I've tried changing from the "close" to "end" event, same result.

Test this script with the following call...
node --max-old-space-size=8192 .\data.load.js 'foo' 'us-west-1' 'dev' '.\foo.default.json' true
Here is the content for the script I ultimately used...
'use strict'
if (process.argv.length < 6) {
throw new Error ('Please pass the table-name, aws-Region, aws-Profile, and file-path to the script.');
}
let [, , TableName, Region, Profile, ImportFile, Force] = process.argv;
process.env.AWS_SDK_LOAD_CONFIG = true;
process.env.AWS_PROFILE = Profile;
Force = typeof(Force) !== 'undefined' ? Force : false;
const AWS = require('aws-sdk');
const fs = require('fs');
const JSONStream = require('JSONStream');
AWS.config.update({ region: Region });
const ddbc = new AWS.DynamoDB.DocumentClient();
console.log('Target Profile: ', Profile);
console.log('Target Region: ', Region);
console.log('Target Table: ', TableName);
console.log('Source File: ', ImportFile);
console.log('Force Import: ', Force);
// Returns the number of records in a specified table
const ddb_table_has_items = (TableName) => {
return new Promise((resolve, reject) => {
const ddb_query_parameters = { TableName, Select: 'COUNT' }
ddbc.scan(ddb_query_parameters, (error, data) => {
(error) ? reject(error) : resolve(data);
});
});
}
const ddb_table_upsert_items = (TableName, Item) => {
return new Promise((reject, resolve) => {
const ddb_insert_payload = { TableName, Item };
ddbc.put(ddb_insert_payload, (error, data) => {
(error) ? reject(error) : resolve(data);
});
});
}
const ddb_bulk_load = (TableName, ImportFile) => {
return new Promise ( (resolve, reject) => {
let count_succeeded = 0;
let count_failed = 0;
let count_attempted = 0;
let inserts = [];
const json_stream = JSONStream.parse( "*" );
const source_data_stream = fs.createReadStream(ImportFile);
const ddb_source_item = source_data_stream.pipe(json_stream);
ddb_source_item.on("data", (source_data_item) => {
count_attempted++;
let ddb_insert = ddb_table_upsert_items(TableName, source_data_item)
.then( (data) => count_succeeded++ )
.catch( (error) => count_failed++ );
inserts.push(ddb_insert);
});
ddb_source_item.on("end", () => {
Promise.all(inserts)
.then(() => {
resolve({count_succeeded, count_failed, count_attempted});
})
.catch((error) => {
console.log(error);
reject(error);
});
});
ddb_source_item.on("error", (error) => {
reject(error);
});
});
}
(async () => {
try {
let proceed_with_import = false;
if ( Force.toString().toLowerCase() === 'true' ) {
proceed_with_import = true;
} else {
const table_scan = await ddb_table_has_items(TableName);
proceed_with_import = ( table_scan.Count === 0 );
}
if (proceed_with_import) {
let ddb_inserts = await ddb_bulk_load(TableName, ImportFile);
console.log("=".repeat(75));
console.log("Completed: '%s' has been loaded into '%s'.", ImportFile, TableName);
console.log(" Insert Attempted: %s", ddb_inserts.count_attempted);
console.log(" Insert Succeeded: %s", ddb_inserts.count_succeeded);
console.log(" Insert Failed : %s", ddb_inserts.count_failed);
}
} catch (error) {
console.log(error);
}
})();
Wrapping each insert in a promise, pushing the insert-promises into an array, and using promise all on that array did the trick. I execute the promise all once we're finished reading from the file; once the "end" event is emitted on the ddb_source_item stream.