I have been learning about the experimental worker threads module in Node.js. I've read the official documentation, as well as most available articles, which are still quite sparse.
I have created a simple example that spawns ten (10) Worker threads in order to generate 10,000 SHA256 digests and then digitally sign them.
Using ten (10) Workers takes around two (2) seconds to generate all 10,000. Without workers, it takes approximately fifteen (15) seconds.
In the official documentation, it states that creating a pool of Workers is recommended versus spawning Workers on demand.
I've tried to find articles on how I'd go about doing this, but I haven't had any luck thus far.
How would I create a pool of Worker threads? Would the worker.js file somehow be modified so that I could create the Workers in advance and then send a message to the workers, which would cause them to execute their code? Would the pool be specific to the use case or is it possible to create a generic pool that could load a file or something and handle any use case?
Thank you.
MAIN
const { performance } = require('perf_hooks')
const { Worker } = require('worker_threads')
// Spawn worker
const spawn = function spawnWorker(workerData) {
return new Promise((resolve, reject) => {
const worker = new Worker('./worker.js', { workerData })
worker.on('message', (message) => resolve(message))
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with exit code ${code}`))
})
})
}
const generate = async function generateData() {
const t0 = performance.now()
const initArray = []
for (step = 1; step < 10000; step += 1000) {
initArray.push({
start: step,
end: step + 999
})
}
const workersArray = initArray
.map(x => spawn(x))
const result = await Promise.all(workersArray)
let finalArray = []
for (let x of result) {
finalArray = finalArray.concat(x.data)
}
const t1 = performance.now()
console.log(`Total time: ${t1 - t0} ms`)
console.log('Length:', finalArray.length)
}
generate()
.then(x => {
console.log('EXITING!')
process.exit(0)
})
WORKERS
const { performance } = require('perf_hooks')
const { workerData, parentPort, threadId} = require('worker_threads')
const crypto = require('crypto')
const keys = require('./keys')
const hash = function createHash(data) {
const result = crypto.createHash('sha256')
result.update(data, 'utf8')
return result.digest('hex')
}
const sign = function signData(key, data) {
const result = crypto.createSign('RSA-SHA256')
result.update(data)
return result.sign(key, 'base64')
}
const t0 = performance.now()
const data = []
for (i = workerData.start; i <= workerData.end; i++) {
const digest = hash(i.toString())
const signature = sign(keys.HTTPPrivateKey, digest)
data.push({
id: i,
digest,
signature,
})
}
const t1 = performance.now()
parentPort.postMessage({
workerData,
data,
time: t1 - t0,
status: 'Done',
})
I would suggest using workerpool. It basically does all the pool management for you and it supports both worker threads and clusters.
Related
Visual Studio Code 1.66.2
ganache v7.1.0 (#ganache/cli: 0.2.0, #ganache
I am new to Solidity and this is my first example I am trying out.
I am trying to get noticed when there is a swap event in the Uniswap Pool "ETH-APE" using:
uPair.events.Swap
The code seems to start where: Waiting for swaps...
I can see on Uniswap itself that there is regular swaps but the swap event seems to not trigger where the console.log should show: someone swapped now!
I have started this blockchain successfully with the command like this:
Notice: I have a real apikey from alchemy and a real mnemonic from ganache-cli
ganache-cli -f wss://eth-mainnet.alchemyapi.io/v2/myAPIkey --mnemonic "word1 word2 word3 etc" -p 7545
I have just followed some examples and are not sure exactly what I am doing:
As I am a beginner I must also ask about ganache. As I have understand this is a "Fake and local Sandboxed" blockchain just existing on my computer?
I think I understand that I start ganache-cli on my computer but are not sure if the uPair.events.Swap listen to my local blockchain which is not the REAL blockchain and this is because the swap event is not triggered. If that is the case then that is what I wonder what I am missing?
(I am not even sure I need ganache to listen to this swap event?)
I am not sure what the alchemyapi.io with myAPIkey is doing in the command to start the local ganache-cli where in the same command I use the mnemonic which has been generated from ganache?
// -- HANDLE INITIAL SETUP -- //
require("dotenv").config();
const Web3 = require('web3')
const IERC20 = require('#openzeppelin/contracts/build/contracts/ERC20.json')
const IUniswapV2Pair = require("#uniswap/v2-core/build/IUniswapV2Pair.json")
const IUniswapV2Factory = require("#uniswap/v2-core/build/IUniswapV2Factory.json")
const { ChainId, Token } = require("#uniswap/sdk")
let web3 = new Web3('ws://127.0.0.1:7545')
const main = async () => {
const _eth_address = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2";
const _apecoin_address = "0x4d224452801aced8b2f0aebe155379bb5d594381";
const _uniswap_factory_address = "0x5C69bEe701ef814a2B6a3EDD4B1652CB9cc5aA6f";
const uFactory = new web3.eth.Contract(IUniswapV2Factory.abi, _uniswap_factory_address) // UNISWAP FACTORY CONTRACT
const { token0Contract, token1Contract, token0, token1 } = await getTokenAndContract(_eth_address, _apecoin_address)
let uPair = await getPairContract(uFactory, token0.address, token1.address)
console.log("Waiting for swaps...");
uPair.events.Swap({}, async () => {
console.log("someone swapped now!");
})
}
async function getPairAddress(_V2Factory, _token0, _token1) {
const pairAddress = await _V2Factory.methods.getPair(_token0, _token1).call()
return pairAddress
}
async function getPairContract(_V2Factory, _token0, _token1) {
const pairAddress = await getPairAddress(_V2Factory, _token0, _token1)
const pairContract = new web3.eth.Contract(IUniswapV2Pair.abi, pairAddress)
return pairContract
}
async function getTokenAndContract(_token0Address, _token1Address) {
const token0Contract = new web3.eth.Contract(IERC20.abi, _token0Address)
const token1Contract = new web3.eth.Contract(IERC20.abi, _token1Address)
const token0 = new Token(
ChainId.MAINNET,
_token0Address,
18,
await token0Contract.methods.symbol().call(),
await token0Contract.methods.name().call()
)
const token1 = new Token(
ChainId.MAINNET,
_token1Address,
18,
await token1Contract.methods.symbol().call(),
await token1Contract.methods.name().call()
)
return { token0Contract, token1Contract, token0, token1 }
}
main()
I need to create a service in NodeJS that periodically executes a GET request to an API that return all the Jobs/Tasks. The service then needs to create a CronJob for each task returned while continuing to check for new tasks, and if there are new ones create new CronJobs.
I made something similar by having a service that runs a GET and then does a forEach loop and creates new CronJobs. But this doesn't take in account new tasks that are created after the first initialization. How do I solve this? How do I make a service that is always looking for new tasks and dynamically creates them?
EDIT1: the axios.post just post a log on a database, nothing special
const axios = require("axios");
const CronJob = require("cron").CronJob;
const cron = require("cron");
const startCron = async () => {
const schedules = await axios
.get("http://127.0.0.1:4000/")
.then((res) => {
return res.data;
})
.catch((err) => console.log(err));
schedules.forEach((schedule) => {
return new CronJob(`${schedule.timing} * * * * *`, () => {
let d = new Date();
console.log(schedule.message + " in data: " + d);
axios.post(`http://127.0.0.1:4000/${schedule.id}`);
}).start();
});
};
startCron();
I have a function that inserts into the database with the POST method and debugging, I test it with postman, sending it an empty post request, so it executes the controller
The function executes 2 more, than 1 is the one that inserts to the DB, ok, I want to execute this function automatically with node-cron
My functions
export class GettingInfo {
ReadingFileFromServer = () => {
const file = path.resolve(__dirname, '../../../dist/entity/PRUEBA.txt')
try {
const data = fs.readFileSync(file, 'utf-8');
const lines = data.split("\n")
let values = []
let bi = []
lines.forEach(line => {
line.trim()
values = line.split("\|", 6).map(a => a.trim());
bi.push(values)
console.log(bi)
})
const convert = this.TransformingFiletoJson(bi)
console.log(convert)
const save = this.SavingReferences(convert)
console.log(save)
} catch (err) {
console.error(err), 'something has happened to the file';
}
}
for the moment to test it I call it in a controller.ts
#Post('data')
createData(){
const tasks = new GettingInfo(this.referenceService)
tasks.ReadingFileFromServer()
return "created! 201 test.."
}
}
But now, that I want to run it alone, create a file "execute.ts" with the following code and it does not run alone
import cron = require("node-cron")
import {GettingInfo} from "./reference.task";
cron.schedule("5 * * * * *", ()=> {
const echale = new GettingInfo(this.referenceService)
echale.ReadingFileFromServer()
console.log("Executing...")
})
From what I can see in the node-cron documentation you need to start the task in order to start the scheduled cron executions.
Change your code to:
const task = cron.schedule("5 * * * * *", ()=> {
const echale = new GettingInfo(this.referenceService)
echale.ReadingFileFromServer()
console.log("Executing...")
})
task.start()
And it should work.
I'm looking for a way to run 3 same-functions at once in a loop and wait until it finish and continues to run another 3 same-functions. I think it involves a loop, promise API. But my solution is fail. It would be great if you could tell me what did I do wrong and how to fix it.
Here is what I have done so far:
I have a download function (call downloadFile), an on-hold function (call runAfter) and a multi download function (call downloadList). They look like this:
const https = require('https')
const fs = require('fs')
const { join } = require('path')
const chalk = require('chalk') // NPM
const mime = require('./MIME') // A small module read Json and turn it to object. It returns a file extension string.
exports.downloadFile = url => new Promise((resolve, reject) => {
const req = https.request(url, res => {
console.log('Accessing:', chalk.greenBright(url))
console.log(res.statusCode, res.statusMessage)
// console.log(res.headers)
const ext = mime(res)
const name = url
.replace(/\?.+/i, '')
.match(/[\ \w\.-]+$/i)[0]
.substring(0, 250)
.replace(`.${ext}`, '')
const file = `${name}.${ext}`
const stream = fs.createWriteStream(join('_DLs', file))
res.pipe(stream)
res.on('error', reject)
stream
.on('open', () => console.log(
chalk.bold.cyan('Download:'),
file
))
.on('error', reject)
.on('close', () => {
console.log(chalk.bold.cyan('Completed:'), file)
resolve(true)
})
})
req.on('error', reject)
req.end()
})
exports.runAfter = (ms, url) => new Promise((resolve, reject) => {
setTimeout(() => {
this.downloadFile(url)
.then(resolve)
.catch(reject)
}, ms);
})
/* The list param is Array<String> only */
exports.downloadList = async (list, options) => {
const opt = Object.assign({
thread: 3,
delayRange: {
min: 100,
max: 1000
}
}, options)
// PROBLEM
const multiThread = async (pos, run) => {
const threads = []
for (let t = pos; t < opt.thread + t; t++) threads.push(run(t))
return await Promise.all(threads)
}
const inQueue = async run => {
for (let i = 0; i < list.length; i += opt.thread)
if (opt.thread > 1) await multiThread(i, run)
else await run(i)
}
const delay = range => Math.floor(
Math.random() * (new Date()).getHours() *
(range.max - range.min) + range.min
)
inQueue(i => this.runAfter(delay(opt.delayRange), list[i]))
}
The downloadFile will download anything from the link given. The runAfter will delay a random ms before excute downloadFile. The downloadList receive a list of URL and pass each of it to runAfter to download. And that (downloadList) is where the trouble begin.
If I just pass the whole list through simple loop and execute a single file at once. It's easy. But if I pass a large requests, like a list with 50 urls. It would take long time. So I decide to make it run parallel at 3 - 5 downloadFile at once, instead of one downloadFile. I was thinking about using async/await and Promise.all to solve the problem. However, it's crash. Below is the NodeJs report:
<--- Last few GCs --->
[4124:01EF5068] 75085 ms: Scavenge 491.0 (493.7) -> 490.9 (492.5) MB, 39.9 / 0.0 ms (average mu = 0.083, current mu = 0.028) allocation failure
[4124:01EF5068] 75183 ms: Scavenge 491.4 (492.5) -> 491.2 (493.2) MB, 29.8 / 0.0 ms (average mu = 0.083, current mu = 0.028) allocation failure
<--- JS stacktrace --->
==== JS stack trace =========================================
0: ExitFrame [pc: 00B879E7]
Security context: 0x03b40451 <JSObject>
1: multiThread [04151355] [<project folder>\inc\Downloader.js:~62] [pc=03C87FBF](this=0x03cfffe1 <JSGlobal Object>,0,0x041512d9 <JSFunction (sfi = 03E2E865)>)
2: inQueue [041513AD] [<project folder>\inc\Downloader.js:70] [bytecode=03E2EA95 offset=62](this=0x03cfffe1 <JSGlobal Object>,0x041512d9 ...
FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
Writing Node.js report to file: report.20200428.000236.4124.0.001.json
Node.js report completed
Apparently, a sub-function of downloadList (multiThread) is a cause but I couldn't read those number (seems like a physical address of RAM or something), so I have no idea how to fix it. I'm not a professional engineer so I would appreciate if you could give me a good explanation.
Addition information:
NodeJs version: 12.13.1
Localhost: Aspire SW3-013 > 1.9GB (2GB in spec) / Intel Atom CPU Z3735F
Connecting to Internet via WiFi (Realtek drive)
OS: Windows 10 (no other choice)
In case you might ask:
Why wrapping Promise for downloadFile? For further application, like I can put it in other app which only require one download at a time.
Does runAfter important? Maybe no, just a little challenges for myself. But it could be useful if servers require delay download time.
Homework or Business? None, hobby only. I plan to build a app to fetch and download image from API of Unsplash. So I prefer a good explanation what I did wrong and how to fix it rather then a code that simple works.
Your for-loop in multiThread never ends because your continuation condition is t < opt.thread + t. This will always be true if opt.thread is not zero. You will have an infinite loop here, and that's the cause of your crash.
I suspect you wanted to do something like this:
const multiThread = async (pos, run) => {
const threads = [];
for (let t = 0; t < opt.thread && pos+t < list.length; t++) {
threads.push(run(pos + t));
}
return await Promise.all(threads);
};
The difference here is that the continuation condition for the loop should be limiting itself to a maximum of opt.thread times, and also not going past the end of the number of entries in the list array.
If the list variable isn't global (ie, list.length is not available in the multiThread function), then you can leave out the second part of the condition and just handle it in the run function like this so that any values of i past the end of the list are ignored:
inQueue(i => {
if (i < list.length) this.runAfter(delay(opt.delayRange), list[i])
})
I have some code for playing with "worker threads" (Node js). When in app.js in loop I set condition i<100000000, my second thread do not start before first thread not finish. How in Node js work sync threads ? And how I can use two and more threads on parallel ?
const { Worker } = require('worker_threads');
const path = require('path');
const WORKERS_NUMBER = 2;
for (var i = 1; i <= WORKERS_NUMBER ; i++) {
const w = new Worker(path.join(__dirname, './app.js'), { workerData: { id: i } });
w.addListener("message",(message)=>{console.log(message);});
}
const { workerData, parentPort } = require('worker_threads');
const id = workerData.id;
console.log(`Worker ${id} initializad.`);
let i=0;
while (i<10) {
i++;
process.nextTick((i)=>{
parentPort.postMessage( `${id}:${i}` );
},i);
}
You need to manually sync the threads' execution.
You can do it by listening on a certain message you'd send from workers to the main thread and using a counter to know when all the threads are done.
Otherwise, you can use the microjob lib that easily wraps the worker thread execution inside a Promise, so the synchronisation becomes trivial.
Take a look to the examples inside the docs.