SCENARIO:
Using mssql I'm connecting to sql and retrieving a list of ids, then based on those id I want to run stored procedures. What I'm currently doing is running the first stored proc, storing the id's in an array, then I'm running a for loop calling another module, where I pass the id to run a stored proc. This works fine when I've got a single id, but fails with 'Global connection already exists. Call sql.close() first.' when I try to run multiple ones.
How do I create connect to sql, run my query, then run the next one? What's the best approach?
The code below runs the stored proc with ids and causes the above error.
exports.runStoredProc = function (query,id) {
sql.connect(config.config).then(()=>{
return sql.query`${query} ${id}`
}).then(res=> {
do something with the response
}).catch(error => {
console.log(error)
})
}
Looks like the connection still exists when the below bit of code runs it using next id. I thought that creating a Promise will force to await execution before it runs the above bit of code again?
let toRun = result.recordset.length
let gen = async num => {
for(let i=0;i<num;i++) {
var resp = result.recordset[i].id
console.log(i, resp)
var sp = report
var reportId = await new Promise(() => db.runStoredProc(sp,resp))
}
}
gen(toRun).then(() => console.log("done!"))
You need to return Promise from runStoredProc
exports.runStoredProc = function (query,id) {
return sql.connect(config.config).then(()=>{
return sql.query`${query} ${id}`
}).then(res=> {
do something with the response
}).catch(error => {
console.log(error)
})
}
and no need to wrap db.runStoredProc in loop
let toRun = result.recordset.length
let gen = async num => {
for(let i=0;i<num;i++) {
var resp = result.recordset[i].id
console.log(i, resp)
var sp = report
var reportId = await db.runStoredProc(sp,resp)
}
}
gen(toRun).then(() => console.log("done!"))
Related
Context
I'm retrieving data from the ESPN API to fetch weekly NFL matchup data. So, I'm making 18 api calls each time I need to fetch this data to account for all 18 weeks in the NFL season. I'm then creating an array with the data I need from the responses to those calls and writing out 18 files that align with each week in the NFL season (week1.json, week2.json, etc.).
Problem
The problem is that when I call my endpoint, I am seeing 2 things intermittently, and not necessarily at the same time:
(1) Some of the json files(week1.json, week2.json, etc.) include only a portion of the expected array. So, instead of 16 objects in the array, I may see only 4, or only 6, etc. Why would I only see a portion of the response data written to the array that's ultimately written to the .json files?
(2) Not all files are written to each time the endpoint is called. So, I may see that only week1-week5's .json files are written. Why aren't all of them updated?
Problem Code
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`)
.then(response => {
const schedule = [];
// get all items from response
const items = response.data.items
// console.log(response.data.items)
items.forEach(item => {
// make get call to $ref
axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
})
.catch(error => {
console.log(error)
})
})
}).catch(error => {
console.log(error)
})
}
Updated Code
router.get('/getschedules', (req, res) => {
async function writeGames() {
// iterate 18 times
for (let i = 0; i < 18; i++) {
let weekNumber;
weekNumber = i + 1;
const week = fs.readFileSync(`./pickem/week${weekNumber}.json`, 'utf8');
const weekJson = JSON.parse(week);
// empty weekJson.games array
weekJson.games = []
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
await axios.get(`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`)
.then(async (response) => { // add async to be able to use await
const schedule = [];
// get all items from response
const items = response.data.items
console.log(response.data.items)
// Use standard loop to be able to benefit from async/await
for (let item of items) {
// make get call to $ref
// wait for an item to be processed before going to the next one
await axios.get(item.$ref)
.then(response => {
// get name
const name = response.data.name
// get date
const date = response.data.date
// get event id
const eventid = response.data.id
// get team ids
let team1 = response.data.competitions[0].competitors[0].id
let team2 = response.data.competitions[0].competitors[1].id
// create new object
const newObject = {
name: name,
date: date,
eventid: eventid,
team1: team1,
team2: team2
}
// add games for week
weekJson.games.push(newObject);
})
.catch(error => {
console.log(error)
})
}
// moved out of the for loop since you only need to write this once
fs.writeFileSync(`./pickem/week${weekNumber}.json`, JSON.stringify(weekJson));
}).catch(error => {
console.log(error)
})
}
}
writeGames();
})
Your issue might come from the fact that you are looping over an array of item that triggers parallel asynchronous calls and write weekJson before you get the entire data. (But theoretically your code should work if writeSyncFile is really synchronous, maybe there are locks on the file system that prevents node to write properly?)
You could try to make everything sequential and only write weekJson once instead of everytime you go over an item:
EDIT
I updated my original code proposition by keeping parallel calls and it worked for me (it's similar to OP's code but I only write the json file once per week).
Then I tried to run OP's code and it was working fine as well. So this makes me think that the problem isn't from the code itself but rather how it's called. As a pure node script, there doesn't seem to be any issue. But I just noticed that OP is using it server side as the result of an API call.
Having an API write so many JSON concurrently is probably not the best idea (especially if the api is called multiple times almost simultaneously). You could either
just return the games in the response
or precompute the results
or fetch and write them only once then cache the result to be reused
Then I wonder if due to the server context, there is not some kind of timeout since OP said that with my initial solution, only the first week was created.
const axios = require("axios");
const fs = require("fs");
async function writeGames() {
const writeWeekGamesPromises = [];
// iterate 18 times
for (let weekNumber = 1; weekNumber < 19; weekNumber++) {
// give week a default value in case the json file doesn't exist (for repro purpose)
let week = "{}";
try {
week = fs.readFileSync(`./pickem/week${weekNumber}.json`, "utf8");
} catch (e) {
console.log(`error reading week ${weekNumber} json file:`, e);
// file doesn't exist yet
}
const weekJson = JSON.parse(week);
// empty weekJson.games array
const games = [];
weekJson.games = games;
// get all items
// Add await keyword to wait for a week to be processed before going to the next one
writeWeekGamesPromises.push(axios
.get(
`https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2022/types/2/weeks/${weekNumber}/events?lang=en®ion=us`
)
.then(async (eventListResponse) => {
// add async to be able to use await
const schedule = [];
console.log(JSON.stringify(eventListResponse.data),'\n');
// get all items from response
const items = eventListResponse.data.items;
// console.log(eventListResponse.data.items); // this seems to be useless since we log the whole data just above
// parallelize calls and wait for all games from a week to be fetched before writing the file
await Promise.all(
items.map((item) => {
// we return the promise so that Promise.all will wait for all games to be pushed before going on writing the file
return axios
.get(item.$ref)
.then((response) => {
// get name, date and eventid
const {name, date, id: eventid} = response.data;
// get team ids
let team1 = response.data.competitions[0].competitors[0].id;
let team2 = response.data.competitions[0].competitors[1].id;
games.push({ name, date, eventid, team1, team2 });
})
.catch((error) => {
console.log(error);
});
})
);
// Now that all game data is ready, write in the file
fs.writeFileSync(
`./pickem/week${weekNumber}.json`,
JSON.stringify(weekJson)
);
})
.catch((error) => {
console.log(error);
}));
}
// Waiting for all games from all weeks to be processed
await Promise.all(writeWeekGamesPromises);
}
async function runAndLogTime() {
const start = Date.now();
await writeGames();
console.log(`took ${(Date.now() - start) / 1000}s to write all json files`);
}
runAndLogTime();
I am new to Neo4j so I quite stuck with looping through some values.
I have a list of skill to skill strings
let data = [
'big_data, business_intelligence',
'big_data, data_collection',
'big_data, economic_growth',
'big_data, economy'
]
And I want to create or update the relation between left side with right side
for (let item of data) {
CreateSkillToSkillRelation(item);
}
const CreateSkillToSkillRelation = async (relation) => {
let mainSkill = relation.split(",")[0];
let secundarySkill = relation.split(",")[1];
try {
// Check if the relationship exists
let { records } = await session.run(
"MATCH(main:SKILL {name:$mainSkill}) -[relation:SKILL_TO_SKILL]-> (secundary:SKILL {name:$secundarySkill}) RETURN relation",
{ mainSkill, secundarySkill }
);
let count =
records[0]?._fields[0].properties.count > 0
? records[0]._fields[0].properties.count + 1
: 1;
// If count is greater then 1 then lets update the counter
if (count > 1) {
await session.run(
"MATCH(main:SKILL {name:$mainSkill}) -[relation:SKILL_TO_SKILL]-> (secundary:SKILL {name:$secundarySkill}) SET relation.count = $count RETURN main, secundary",
{
mainSkill,
secundarySkill,
count,
}
);
}
// Otherwise the skill relation is not created so lets create one
else {
await session.run(
"CREATE(main:SKILL {name:$mainSkill}) -[:SKILL_TO_SKILL {count:$count}]-> (secundary:SKILL {name:$secundarySkill}) RETURN main, secundary",
{
mainSkill,
secundarySkill,
count,
}
);
}
await session.close();
} catch (error) {
console.log(error);
}
};
But every time when I run this I get the following error Neo4jError: Queries cannot be run directly on a session with an open transaction; either run from within the transaction or use a different session.
Any idea how can I solve this?
for (let item of data) {
CreateSkillToSkillRelation(item);
}
Is not awaiting the promises you create and so you are basically trying to run all of these promises concurrently against a single session which only supports a single concurrent transaction.
You should create a session in each call of CreateSkillToSkillRelation or await each call to it using a single session.
Though note you close the session at the end of CreateSkillToSkillRelation but only on success, might I suggest moving await session.close(); into a finally block.
The answer of the colleague #just_another_dotnet_dev is absolutely correct: you run asynchronous functions in a loop, and close the session in one of them.
The Cipher language is very rich, and you can use it to do everything that you tried to do with a loop in Javascript. Something like this, using UNWIND and MERGE:
const CreateSkillToSkillRelations = async (data) => {
const session = driver.session();
try {
let { records } = await session.run(
`WITH split(row, ',') as rels
WITH trim(rels[0]) as mainSkill,
trim(rels[1]) as secundarySkill
MERGE (main:SKILL {name: mainSkill})
-[relation:SKILL_TO_SKILL]->
(secundary:SKILL {name: secundarySkill})
ON CREATE SET relation.count = 1
ON MATCH SET relation.count = relation.count + 1
RETURN main, relation, secundary`,
{ data }
);
} catch (error) {
console.log(error);
} finally {
await session.close()
}
};
Problem Statement:
Our aim is to allocate values in the array ytQueryAppJs, which are returned from a time consuming function httpsYtGetFunc().
The values in ytQueryAppJs needs to be used many times in further part of the code, hence it needs to be done 'filled', before the code proceeds further.
There are many other arrays like ytQueryAppJs, namely one of them is ytCoverAppJs, that needs to be allocated the value, the same way as ytQueryAppJs.
The values in ytCoverAppJs further require the use of values from ytQueryAppJs. So a solution with clean code would be highly appreciated.
(I am an absolute beginner. I have never used async, await or promises and I'm unaware of the correct way to use it. Please guide.)
Flow (to focus on):
The user submits a queryValue in index.html.
An array ytQueryAppJs is logged in console, based on the query.
Expected Log in Console (similar to):
Current Log in Console:
Flow (originally required by the project):
User submits query in index.html.
The values of arrays, ytQueryAppJs, ytCoverAppJs, ytCoverUniqueAppJs, ytLiveAppJs, ytLiveUniqueAppJs gets logged in the console, based on the query.
Code to focus on, from 'app.js':
// https://stackoverflow.com/a/14930567/14597561
function compareAndRemove(removeFromThis, compareToThis) {
return (removeFromThis = removeFromThis.filter(val => !compareToThis.includes(val)));
}
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
let promise = new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = []
response.on('data', (d) => {
chunks.push(d)
})
// Parsing the chunks
response.on('end', () => {
resultOfYtGetFunc = JSON.parse((Buffer.concat(chunks).toString()))
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < (resultOfYtGetFunc.items).length; i++) {
extractedResultOfYtGetFunc[i] = resultOfYtGetFunc.items[i].id.videoId;
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
})
})
})
let result = await promise;
return result;
}
app.post("/", function(req, res) {
// Accessing the queryValue, user submitted in index.html. We're using body-parser package here.
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Complete app.post method from app.js:
(For better understanding of the problem.)
app.post("/", function(req, res) {
// Accessing the queryValue user submitted in index.html.
query = req.body.queryValue;
// Fetcing top results related to user's query and putting them in the array.
ytQueryAppJs = httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
// Fetching 'cover' songs related to user's query and putting them in the array.
if (query.includes("cover") == true) {
ytCoverAppJs = httpsYtGetFunc(query);
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
} else {
ytCoverAppJs = httpsYtGetFunc(query + " cover");
console.log("ytCoverAppJs:");
console.log(ytCoverAppJs);
// Removing redundant values.
ytCoverUniqueAppJs = compareAndRemove(ytCoverAppJs, ytQueryAppJs);
console.log("ytCoverUniqueAppJs:");
console.log(ytCoverUniqueAppJs);
}
// Fetching 'live performances' related to user's query and putting them in the array.
if (query.includes("live") == true) {
ytLiveAppJs = httpsYtGetFunc(query);
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
} else {
ytLiveAppJs = httpsYtGetFunc(query + " live");
console.log("ytLiveAppJs:");
console.log(ytLiveAppJs);
// Removing redundant values.
ytLiveUniqueAppJs = compareAndRemove(ytLiveAppJs, ytQueryAppJs.concat(ytCoverUniqueAppJs));
console.log("ytLiveUniqueAppJs:");
console.log(ytLiveUniqueAppJs);
}
// Emptying all the arrays.
ytQueryAppJs.length = 0;
ytCoverAppJs.length = 0;
ytCoverUniqueAppJs.length = 0;
ytLiveAppJs.length = 0;
ytLiveUniqueAppJs.length = 0;
});
Unfortunately you can use the async/await on http module when making requests. You can install and use axios module . In your case it will be something like this
const axios = require('axios');
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
async function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI"
urlOfYtGetFunc = "https://www.googleapis.com/youtube/v3/search?key=" + apiKey + "&part=snippet&q=" + queryOfYtGetFunc + "&maxResults=4&order=relevance&type=video";
const promise = axios.get(urlOfYtGetFunc).then(data => {
//do your data manipulations here
})
.catch(err => {
//decide what happens on error
})
Or async await
const data = await axios.get(urlOfYtGetFunc);
//Your data variable will become what the api has returned
If you still want to catch errors on async await you can use try catch
try{
const data = await axios.get(urlOfYtGetFunc);
}catch(err){
//In case of error do something
}
I have just looked at the code I think the issue is how you are handling the async code in the request handler. You are not awaiting the result of the function call to httpsYtGetFunc in the body so when it returns before the promise is finished which is why you get the Promise {Pending}.
Another issue is that the array is not extractedResultOfYtGetFunc is not initialised and you may access indexes that don't exist. The method to add an item to the array is push.
To fix this you need to restructure your code slightly. A possible solution is something like this,
// Declaring variables for the function 'httpsYtGetFunc'
let apiKey = "";
let urlOfYtGetFunc = "";
let resultOfYtGetFunc = "";
let extractedResultOfYtGetFunc = [];
// This function GETs data, parses it, pushes required values in an array.
function httpsYtGetFunc(queryOfYtGetFunc) {
apiKey = "AI...MI";
urlOfYtGetFunc =
"https://www.googleapis.com/youtube/v3/search?key=" +
apiKey +
"&part=snippet&q=" +
queryOfYtGetFunc +
"&maxResults=4&order=relevance&type=video";
return new Promise((resolve, reject) => {
// GETting data and storing it in chunks.
https.get(urlOfYtGetFunc, (response) => {
const chunks = [];
response.on("data", (d) => {
chunks.push(d);
});
// Parsing the chunks
response.on("end", () => {
// Initialising the array
extractedResultOfYtGetFunc = []
resultOfYtGetFunc = JSON.parse(Buffer.concat(chunks).toString());
// console.log(resultOfYtGetFunc)
// Extracting useful data, and allocating it.
for (i = 0; i < resultOfYtGetFunc.items.length; i++) {
// Adding the element to the array
extractedResultOfYtGetFunc.push(resultOfYtGetFunc.items[i].id.videoId);
// console.log(extractedResultOfYtGetFunc);
}
resolve(extractedResultOfYtGetFunc);
});
});
});
}
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
ytQueryAppJs = await httpsYtGetFunc(query);
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
});
Another option would be to use axios,
The code for this would just be,
app.post("/", async function (req, res) {
query = req.body.queryValue;
// Fetching top results related to user's query and putting them in the array.
try{
ytQueryAppJs = await axios.get(url); // replace with your URL
console.log("ytQueryAppJs:");
console.log(ytQueryAppJs);
} catch(e) {
console.log(e);
}
});
Using Axios would be a quicker way as you don't need to write promise wrappers around everything, which is required as the node HTTP(S) libraries don't support promises out of the box.
Good Afternoon,
I am using the MERN stack to making a simple invoice application.
I have a function that runs 2 forEach() that goes through the invoices in the DB and the Users. if the emails match then it gives the invoices for that user.
When I log DBElement to the console it works, it has the proper data, but when I log test1 to the console (app.get()) it only has one object not both.
// forEach() function
function matchUserAndInvoice(dbInvoices, dbUsers) {
dbInvoices.forEach((DBElement) => {
dbUsers.forEach((userElement) => {
if(DBElement.customer_email === userElement.email){
const arrayNew = [DBElement];
arrayNew.push(DBElement);
app.set('test', arrayNew);
}
})
})
}
// end point that triggers the function and uses the data.
app.get('/test', async (req,res) => {
const invoices = app.get('Invoices');
const users = await fetchUsersFromDB().catch((e) => {console.log(e)});
matchUserAndInvoice(invoices,users,res);
const test1 = await app.get('test');
console.log(test1);
res.json(test1);
})
function matchUserAndInvoice(dbInvoices, dbUsers) {
let newArray = [];
dbInvoices.forEach((DBElement) => {
dbUsers.forEach(async(userElement) => {
if(DBElement.customer_email === userElement.email){
newArray.push(DBElement);
app.set('test', newArray);
}
})
})
}
app.set('test', DBElement); overrides the existing DBElement, so only the last matching DBElement is shown in test1.
If you want to have test correspond to all matching DBElement, you should set it to an array, and then append a new DBElement to the array each time it matches inside the for-loop:
if(DBElement.customer_email === userElement.email){
let newArray = await app.get('test');
newArray.push(DBElement);
app.set('test', newArray);
}
I'm trying to iterate and print out in order an array in Javascript that contains the title of 2 events that I obtained from doing web scraping to a website but it prints out in disorder. I know Javascript is asynchronous but I'm new in this world of asynchronism. How can I implement the loop for to print the array in order and give customized info?
agent.add('...') is like console.log('...'). I'm doing a chatbot with DialogFlow and NodeJs 8 but that's not important at this moment. I used console.log() in the return just for debug.
I tried the next:
async function printEvent(event){
agent.add(event)
}
async function runLoop(eventsTitles){
for (let i = 0; i<eventsTitles.length; i++){
aux = await printEvent(eventsTitles[i])
}
}
But i got this error error Unexpected await inside a loop no-await-in-loop
async function showEvents(agent) {
const cheerio = require('cheerio');
const rp = require('request-promise');
const options = {
uri: 'https://www.utb.edu.co/eventos',
transform: function (body) {
return cheerio.load(body);
}
}
return rp(options)
.then($ => {
//** HERE START THE PROBLEM**
var eventsTitles = [] // array of event's titles
agent.add(`This mont we have these events available: \n`)
$('.product-title').each(function (i, elem) {
var event = $(this).text()
eventsTitles.push(event)
})
agent.add(`${eventsTitles}`) // The array prints out in order but if i iterate it, it prints out in disorder.
// *** IMPLEMENT LOOP FOR ***
agent.add(`To obtain more info click on this link https://www.utb.edu.co/eventos`)
return console.log(`Show available events`);
}).catch(err => {
agent.add(`${err}`)
return console.log(err)
})
}
I would like to always print out Event's title #1 and after Event's title #2. Something like this:
events titles.forEach((index,event) => {
agent.add(`${index}. ${event}`) // remember this is like console.log(`${index}. ${event}`)
})
Thanks for any help and explanation!
There no async case here but if you still face difficultly than use this loop
for (let index = 0; index < eventsTitles.length; index++) {
const element = eventsTitles[index];
agent.add(${index}. ${element})
}