I have a function that inserts into the database with the POST method and debugging, I test it with postman, sending it an empty post request, so it executes the controller
The function executes 2 more, than 1 is the one that inserts to the DB, ok, I want to execute this function automatically with node-cron
My functions
export class GettingInfo {
ReadingFileFromServer = () => {
const file = path.resolve(__dirname, '../../../dist/entity/PRUEBA.txt')
try {
const data = fs.readFileSync(file, 'utf-8');
const lines = data.split("\n")
let values = []
let bi = []
lines.forEach(line => {
line.trim()
values = line.split("\|", 6).map(a => a.trim());
bi.push(values)
console.log(bi)
})
const convert = this.TransformingFiletoJson(bi)
console.log(convert)
const save = this.SavingReferences(convert)
console.log(save)
} catch (err) {
console.error(err), 'something has happened to the file';
}
}
for the moment to test it I call it in a controller.ts
#Post('data')
createData(){
const tasks = new GettingInfo(this.referenceService)
tasks.ReadingFileFromServer()
return "created! 201 test.."
}
}
But now, that I want to run it alone, create a file "execute.ts" with the following code and it does not run alone
import cron = require("node-cron")
import {GettingInfo} from "./reference.task";
cron.schedule("5 * * * * *", ()=> {
const echale = new GettingInfo(this.referenceService)
echale.ReadingFileFromServer()
console.log("Executing...")
})
From what I can see in the node-cron documentation you need to start the task in order to start the scheduled cron executions.
Change your code to:
const task = cron.schedule("5 * * * * *", ()=> {
const echale = new GettingInfo(this.referenceService)
echale.ReadingFileFromServer()
console.log("Executing...")
})
task.start()
And it should work.
Related
I really need some help, I'm new to coding and I'm trying to make a script
The script is supposed to achieve the following:
Takes a picture
Finds text within the image using tesseract
Search for a specific string within the text founded
Preforms an action based on if the specific string has been found or not
The problem I am having is that every time I run the script, it uses the previous version of the image saved, giving me the wrong result at the time.
I could really use some help.
const robot = require('robotjs')
const Jimp = require('jimp')
const Tesseract = require('tesseract.js');
const { Console, log } = require("console");
const fs = require('fs');
const {readFileSync, promises: fsPromises} = require('fs');
const { resolve } = require('path');
const myLogger = new Console({
stdout: fs.createWriteStream("normalStdout.txt")
});
const myLogger2 = new Console({
stdout: fs.createWriteStream("normalStdout2.txt")
});
//////////////////////////////////////////////////////////////////////////////////////////
function main(){
sleep(2000);
performRead();
}
//Edited function to sync instead of async - The problem is still persisting
//Edited function to include tesseractimage() in callback of writeimage()
function writeImage(){
var width = 498;
var height = 135;
var img4 = robot.screen.capture(0, 862, width, height).image;
new Jimp({data: img4, width, height}, (err, image) => {
image.write("image.png", function() {
tesseractimage();
});
});
console.log("Image 1 created");
}
function tesseractimage(){
Tesseract.recognize("image.png", 'eng')
.then(out => myLogger.log(out));
//Saves image to normalstdOut.txt
console.log("Tesseracted image")
}
function readTest(normalStdout, Viverz) {
var path = require('path');
const contents = readFileSync(path.resolve("normalStdout.txt"), 'utf-8');
const result = contents.includes("Viverz");
console.log(result);
}
//Edited performRead removing the call for tesseractimage();, it is now in writeimage();
function performRead(){
writeImage();
readTest();
}
function sleep(ms){
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
return null;
}
main();
I have tried changing functions to async functions,
I've tried numerous methods, pauses,
reiterations of functions multiple times,
nothing saves the file until the script ends and
then after it finds the correct string from the
previously saved screenshot, not the new one.
Current output:
Image 1 created a false Tesseracted image
Even when forcing tesseractimage() to call before the result is published it still has the same problem of not reading the file until the script is over
One way to call tesseractimage() from writeImage() when using image.write():
new Jimp({data: img4, width, height}, (err, image) => {
image.write("image.png", function() {
tesseractimage();
});
});
One way to call tesseractimage() from writeImage() when using image.writeAsync():
new Jimp({data: img4, width, height}, (err, image) => {
image.writeAsync("image.png")
.then((result) => {
tesseractimage();
}).catch((error) => {
// Handle error
})
});
Also remove the function call from within performRead().
For reference look under "Writing to files and buffers".
Solved**
I removed the readTest() altogether, and restructured the tesseractimage to a new function
async function tesseracttest() {
const finalText = await Tesseract.recognize(
"image.png",
'eng',
{ logger: m => console.log(m) }
).then(({ data: { text } }) => {
let extractedText = text.toString();
let finalText = extractedText.includes("Prayer potion");
console.log(extractedText)
console.log(finalText);
return finalText;
});
}
I need to create a service in NodeJS that periodically executes a GET request to an API that return all the Jobs/Tasks. The service then needs to create a CronJob for each task returned while continuing to check for new tasks, and if there are new ones create new CronJobs.
I made something similar by having a service that runs a GET and then does a forEach loop and creates new CronJobs. But this doesn't take in account new tasks that are created after the first initialization. How do I solve this? How do I make a service that is always looking for new tasks and dynamically creates them?
EDIT1: the axios.post just post a log on a database, nothing special
const axios = require("axios");
const CronJob = require("cron").CronJob;
const cron = require("cron");
const startCron = async () => {
const schedules = await axios
.get("http://127.0.0.1:4000/")
.then((res) => {
return res.data;
})
.catch((err) => console.log(err));
schedules.forEach((schedule) => {
return new CronJob(`${schedule.timing} * * * * *`, () => {
let d = new Date();
console.log(schedule.message + " in data: " + d);
axios.post(`http://127.0.0.1:4000/${schedule.id}`);
}).start();
});
};
startCron();
I am just trying a simple get command with Firestore, using this code from Google it doesn't work because it's not waiting for the promise?
Earlier I had put only a snippet of code, this is the entirety of index.js -- I'm using Firestore with Dialogflow to build a Google Assistant app and trying to call a function from the welcome intent that gets a field from Firestore, then writes that field to a string (named question1), and then this string should be spoken by the assistant as part of the ssml response. I've been on this for at least 30 hours already, can't seem to comprehend promises in regards to intents, firestore, etc. I've tried about 10 different solutions, this one works, only it says "undefined" in other variations I have tried it would say undefined several times but after 2-3 passes the get command would be complete and then the variable would be read out. I'm just trying to figure out how to get the get command and variable set before moving onto the SSML response. Can anyone point me in the right direction?
'use strict';
const functions = require('firebase-functions'); //don't forget this one
// Import Admin SDK
var admin = require("firebase-admin");
admin.initializeApp(functions.config().firebase);
var db = admin.firestore();
const collectionRef = db.collection('text');
const Firestore = require('#google-cloud/firestore');
var doc;
var question1;
const url = require('url');
const {
dialogflow,
Image,
Permission,
NewSurface,
} = require('actions-on-google');
const {ssml} = require('./util');
const config = functions.config();
const WELCOME_INTENT = 'Default Welcome Intent';
const app = dialogflow({debug: true});
async function dbaccess(rando) {
console.log("dbaseaccess started")
var currentquestion2 = 'question-' + rando.toString();
var cityRef
try { return cityRef = db.collection('text').doc(currentquestion2).get();
console.log("get command completed")
//do stuff
question1 = cityRef.data().n111
} catch(e) {
//error!
}
console.log("one line above return something");
return rando;
}
app.fallback((conv) => {
// intent contains the name of the intent
// you defined in the Intents area of Dialogflow
const intent = conv.intent;
switch (intent) {
case WELCOME_INTENT:
var rando = Math.floor(Math.random() * 3) + 1;
dbaccess(rando);
const ssml =
'<speak>' +
question1 +
'</speak>';
conv.ask(ssml);
break;
exports.dialogflowFirebaseFulfillment = functions.https.onRequest(app);
You have 2 options: you can use async/await or you can use Promise.then() depending on how you want the code to execute.
Async/await:
async function databasetest {
var cityRef;
try{
cityRef = await db.collection('cities').doc('SF');
// do stuff
} catch(e) {
// error!
}
Promise.then():
db.collection('cities').doc('SF').then((cityRef) => {
cityRef.get()
.then(doc => { /* do stuff */ })
.catch(err => { /* error! */ });
});
maybe a little of work around could help you, I'm not sure yet how you are trying to implement it.
function databasetest () {
var cityRef = db.collection('cities').doc('SF');
return cityRef.get()
}
// so you can implement it like
databasetest().then(doc => {
if (!doc.exists) {
console.log('No such document!');
} else {
console.log('Document data:', doc.data());
}
})
.catch(err => {
console.log('Error getting document', err);
});
More context would help to understand your use case better :)
I have multiple scrapers like below:
await scraper1.run
await scraper2.run
// etc
to increase performance and response time I used websocket and I pass socket connection down to each scraper and emit for each single item (of the result).
const express = require('express')
const app = express()
const http = require('http').Server(app)
const cors = require('cors')
const puppeteer = require('puppeteer')
const io = require('socket.io')(http)
const mongoose = require('mongoose')
const _ = require('lodash')
const scraper1 = require('./scraper1')
const scraper2 = require('./scraper2')
mongoose.connect("mongodb://localhost:27017/test")
;(async function () {
try {
const browser = await puppeteer.launch({
headless: false
})
io.on('connection', async function (socket) {
socket.on('search', async function (query) {
// check whether document exists with user ip address then return
// otherwise run the scrapres
await scraper1.run(browser, socket, query)
await scraper2.run(browser, socket, query)
})
})
} catch (e) {
console.log(e)
}
})()
http.listen(3000)
Context: When user does refresh and multiple socket connections made the scrapers run multiple times and data become duplicate. I prevented the duplication using mongodb but the performance issue remains because scrapers run until their result are ready then I check with the database.
Question: How to lock or prevent scrapers from running multiple times and also wait for each scraper to be done with websocket?
I can propose you next solution (I didn't test it but I guess you can figure out what I am trying to achieve, the explanation below the example):
'use strict';
const queryState = {
};
const getQueryKey = (query) => {
// base64 but can be a hash like sha256
const key = Buffer.from(query).toString('base64');
return key;
};
/**
* Return query state
* #param {String} query
* #return {String} state [PENDING, DONE, null] null if query doesn't exist
*/
const getQueryState = (query) => {
const key = getQueryKey(query);
const state = queryState[key] || null;
return state;
};
/**
* Add a query and initialize it as pending
* #param {String} query
* #return {String} state
*/
const addQuery = (query) => {
const key = getQueryKey(query);
const state = 'PENDING';
queryState[key] = state;
return state;
};
/**
* Hashmap to associate pending queries to be notified to socket connections
* when query is done
* This structure keeps and array of callbacks per query key
*/
const observers = {
};
const addObserver = (query, callback) => {
const key = getQueryKey(query);
if (typeof observers[key] !== 'undefined') {
observers[key] = [callback];
} else {
observers[key] = [...observers[key], callback];
}
};
const notifyObservers = (query) => {
const key = getQueryKey(query);
const callbacks = observers[key] || [];
// TODO: get query data scrapper from a cache / database / etc
const data = getDataFromQuery(query);
callbacks.forEach((callback) => {
callback(data);
});
};
/**
* Update query status to done
* PreCD: query must exist in queryState (previously added using addQuery)
* #param {String} query
* #return {String} state
*/
const endQuery = (query) => {
const key = getQueryKey(query);
const state = 'DONE';
queryState[key] = state;
return state;
};
io.on('connection', async function (socket) {
socket.on('search', async function (query) {
/**
* If query doesn't exist, scrap it
*/
const state = getQueryState(query);
if (state === null) {
addQuery(query);
await scraper1.run(browser, socket, query);
await scraper2.run(browser, socket, query);
endQuery(query);
// store scrapper data in cache / database / etc and
// socket send scraperData to the user
// notify pending queries to send data scrapper
notifyObservers(query);
} else if (state === 'PENDING') {
// add callback to return data to the user
addObserver(query, (scraperData) => {
// socket send scraperData to the user
});
} else {
// socket send scraperData to the user
}
});
});
To simplify the things, the example is simple but not the best one in terms of performance / architecture. This solution implements:
Scenario 1 (someone asking for query1 first time)
A request (socket connection) came to the backend asking for a query
This request doesn't exist yet, so, mark it as PENDING and start scrappers
Scenario 2 (another one asking for query1)
A second connection appears asking for same query as scenario 1
The request is in state PENDING, so, we add a callback to be call when this query finish
Scenario 3 (query 1 finish)
Scrappers started at Scenario 1 ends so query1 is market as DONE
Each request (observer) waiting for query1 will be notified
This solution can have multiple ways to be implemented, but my point is try to expose this to you and then you can modify it in the way you want.
Hope this helps
I have a simple function to where I get the word count from an url. The script works if I have a low amount of urls. I only limit async 4 at a time. I watch my ram and cpu and it doesn't go near the max on my machine. Let's say after about 70ish urls there is no error. The script just sets there. I have it in a try catch block and it never catches. Any help would be appreciated.
I have tried lodash forEach instead of async and I get the same issue.
const async = require('async')
const wordcount = require('wordcount')
const afterLoad = require('after-load')
const htmlToText = require('html-to-text')
function getWordCount(urls, cb) {
async.eachLimit(urls, 4, function(url, cbe) {
try {
let html = afterLoad(url) // https://www.npmjs.com/package/after-load
let text = htmlToText.fromString(html)
let urlWordCount = wordcount(text) // https://www.npmjs.com/package/wordcount
console.log(url, urlWordCount)
cbe(null)
} catch(err) {
console.log(err)
urlWordCount = 0
console.log(url, urlWordCount, err)
cbe(null)
}
}, function(err) {
console.log("finished getting wordcount", err)
if (err) {
cb(err)
} else {
cb(null)
}
})
}
getWordCount(["https://stackoverflow.com/", "https://caolan.github.io/async/docs.html#eachLimit"], function(err){
console.log(err)
})
I think the issue is in the synchronous implementation of that after-load module, but it's indeed hard to judge unless you get an actual error (you could put some console.logs here and there on every line and see where your code actually gets stuck - or use a debugger for the same purpose).
What I'd propose though is to use proper asynchronous code - I run the example below with a set of 1000 urls and it did not get stuck - with usage of [scramjet] it's also more readable:
const {StringStream} = require('scramjet');
const wordcount = require('wordcount');
const fetch = require('node-fetch');
const htmlToText = require('html-to-text');
const {promisify} = require('util');
StringStream.fromArray(["https://stackoverflow.com/", "https://caolan.github.io/async/docs.html#eachLimit"])
.setOptions({maxParallel: 4})
.parse(async url => ({
url,
response: await fetch(url)
}))
.map(async ({url, response}) => {
const html = await response.text();
const text = htmlToText.fromString();
const count = wordcount(text);
return {
url,
count
};
})
.each(console.log)
;
I actually run this from a file with the URL's by changing the first lines to:
StringStream.from(fs.createReadStream('./urls-list.txt'), 'utf-8')
.lines()
.setOptions({maxParallel: 4})
// and so on.