I have a folder that has some json files in it, so I want to access them through my backend dynamically without the hassle to hard code changing data.
The Code works except one thing
localhost/Foo.json
Prints {Name:Bar} => expected Foo
localhost/Bar.json
Prints {Name:Bar} => expected Bar
so always the last entry in the data array is shown
interface filesPath{
name:string,
ext:string, //here .json
filepath:string,
}
var generateJson = (files:filesPath[]) => {
var dataFiles = [];
var dataNames = [];
for(var x in files){
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,(req, res) => {
res.json(dataFiles[x]) //Returns always last entry in array
});
}
}
Is there a simple fix?
Thanks in advance
You've stumbled upon the concept of closures. What is happening is that the x you are passing to your function is not a copy of the value of x but the variable x itself (kind of like a pointer). This means all functions share the same variable thus one variable can only have one value.
The classical way to solve this is to pass x as an argument, thus breaking the closure:
for(var x in files){
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,((new_x) => (req, res) => {
res.json(dataFiles[new_x])
})(x)); // <-- pass in x here to become new_x
}
ES6 introduced the concept of let which does not create a closure. So a more modern solution is to simply use let instead of var:
for(let x in files){ // <-- this fixes it!!
dataFiles[x] = require(files[x].filepath);
dataNames[x] = files[x].name;
app.get('/'+dataNames[x]+files[x].ext,(req, res) => {
res.json(dataFiles[x])
});
}
Related
So i'm attempting to write a google parser.
The idea of my tool is it takes search queries and searches google for them and returns URLs. It is working good so far but now im trying to set a page configuration and im having troubles, my code is:
const needle = require("needle") //for making get request
const sp = require("serp-parser") //for parsing data from the request
const queryup = "watch movies online free" //my search data
const query = encodeURI(queryup) //my search data so google can read it
var page = 0; //initializing the page counter
let pages = 5; //setting amount of pages to loop through
for (var i = 0; i < pages; i++) { //my loop
needle.get(`https://www.google.com/search?q=${query}&start=${page}`, function(err, response){ //MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
page += 10 //adding to page value (every 10 page value is 1 extra page)
console.log(`----- Page number: `+ page / 10+" -----") //logging the number of the page to confirm that it is indeed increasing the page value
let results = response.body; //defining the body of my request
parser = new sp.GoogleNojsSERP(results); //initializing the parser
let parsed = parser.serp //parsing the body
let objarray = parsed.organic; //parsed body (returns as an array of json objects)
for (var i = 0; i < objarray.length; i++) { //loop the logging of each url
let url = objarray[i].url //defining url
console.log(url) //logging each url
}
});
}
without a billion comments:
const needle = require("needle")
const sp = require("serp-parser")
const queryup = "watch movies online free"
const query = encodeURI(queryup)
var page = 0;
let pages = 5;
for (var i = 0; i < pages; i++) {
needle.get(`https://www.google.com/search?q=${query}&start=${page}`, function(err, response){
//^^^^^ MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
page += 10
console.log(`----- Page number: `+ page / 10+" -----")
let results = response.body;
parser = new sp.GoogleNojsSERP(results);
let parsed = parser.serp
let objarray = parsed.organic;
for (var i = 0; i < objarray.length; i++) {
let url = objarray[i].url
console.log(url)
}
});
}
This seems to be an issue with async.
I'm not familiar with needle, but I know that external queries are basically never synchronous.
The problem you're experiencing is basically, the actual web query is happening after the loop first runs and has already incremented page to 50. Then, 5 queries are constructed, each one with page=50, because async is complicated and difficult to manage.
Under the hood, the engine is essentially doing literally everything else it can possibly do first, and THEN doing your web queries.
A trip through the needle npm docs tells me that you can use alternative syntax to get needle to return a promise instead, which can then be wrapped in an asynchronous function and managed through await to force synchronous behavior, which is what you're after:
const needle = require('needle');
const sp = require('serp-parser');
const queryup = 'watch movies online free';
const query = encodeURI(queryup);
let page = 0;
const pages = 5;
const googler = async function () {
for (let i = 0; i < pages; i++) {
try {
const response = await needle('get', `https://www.google.com/search?q=${query}&start=${page}`);// MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
console.log('----- Page number: ' + page / 10 + ' -----');
const results = await response.body;
const parser = new sp.GoogleNojsSERP(results);
const parsed = parser.serp;
const objarray = parsed.organic;
for (let i = 0; i < objarray.length; i++) {
const url = objarray[i].url;
console.log(url);
}
} catch (err) {
console.error(err);
}
page += 10;
}
};
googler();
The key differences:
Per the needle docs, rather than the request method being a method on the needle object, it's instead the first argument you pass directly to invoking needle itself as a function.
When you manage promises with await, a rejected promise throws an error that should be caught with a traditional try/catch block; I've done that here. Though, if needle is anything like node-fetch it probably basically never throws errors, but it's good practice.
One of my extensions automatically changed your var declarations to let and not-reassigned let declarations to const; you're welcome to change them back.
This is a classic asynchronous problem. Add another console.log() immediately before the needle.get() call (and after the for statement) and you will see what is going wrong: All of the needle.get() calls execute before any of the callbacks where you do the page += 10. Then, after the for loop completes, all of the callbacks are executed. But it is too late for this to have any effect on the start= parameter.
One way to fix this could be to move the body of this for loop (the needle.get() and its callback) into a separate function. Initialize your variables and call this function once. Then at the end of the callback, do your page += 10 and update any other variables you need to, and call this function again from there if there are more pages left that you want to load. If you have completed all of the pages, then don't make that call. The for loop is not needed with this technique.
Or, you could keep your current code but move the page += 10 after the callback but still inside the outer for loop. That way this variable will be incremented as you expect. I don't necessarily recommend this, as Google may get unhappy about receiving the get requests so rapidly and may start blocking your calls or throwing CAPTCHAs at you.
There may be an issue of whether this kind of scraping is allowed by Google's Terms of Service, but I will leave that question to you and your legal advisors.
Also, I would avoid using var anywhere. Use const or let instead, and prefer const over let except when you need to reassign the variable.
One tip: in most cases where you use a numeric for loop to iterate over an array, the code will be cleaner if you use a for..of loop. For example, this bit of code:
let parsed = parser.serp
let objarray = parsed.organic;
for (var i = 0; i < objarray.length; i++) {
let url = objarray[i].url
console.log(url)
}
could be more simply written as:
for (const result of parser.serp.organic) {
console.log(result.url)
}
(I know that is just a bit of debug code, but this is a good habit to get into.)
Finally, watch your indentation and be sure to indent nested blocks or functions. I took the liberty of adding some indentation for you.
im on javascript and im currently trying to work with pull requests, issues and commits from a repo. I have the following code:
const axios = require('axios');
var gitPullApiLink = "https://api.github.com/repos/elixir-lang/elixir/pulls";
var listOfCommits = [];
var listOfSHAs = [];
var mapOfInfoObjects = new Map();
var mapPullRequestNumberToCommits = new Map();
var mapPRNumbersToCommitObjects = new Map();
var listOfPrObjects = [];
var setOfFileObjects = new Set();
var listOfNumbersOfTargetedIssues = [];
var mapPRnumberToCloseOpenDateObjects = new Map();
class PullRequestParser {
async getListOfPullRequests(pullrequestLink) {
const message = await axios.get(pullrequestLink);
//console.log(message);
listOfPrObjects = message['data'];
}
async getCommitsForEachPullRequestAndPRinformation() {
var listOfPrNumbers = [];
var k;
// this loop will just make a list of Pull Request Numbers
for (k = 0; k < listOfPrObjects.length; k++){
var currPrNumber = listOfPrObjects[k]['number'];
listOfPrNumbers.push(currPrNumber);
}
// I created a separate list just because... I did it this way because on the github API website it seems
// like the pull request has the same number as the issue it affects. I explain how you can see this down below
listOfNumbersOfTargetedIssues = listOfPrNumbers;
// next loop will make objects that contain information about each pull request.
var n;
for (n = 0; n < listOfPrNumbers; n++){
var ApiLinkForEachPullRequest = gitPullApiLink + "/" + listOfPrNumbers[n];
const mes = await axios.get(ApiLinkForEachPullRequest);
var temp = {OpeningDate: mes['data']['created_at'],
ClosingDate: mes['data']['closed_at'],
IssueLink: mes['data']['_links']['issue']['href']};
//mapPRnumberToCloseOpenDateObjects will be a map where the key is the pull request number and the value
// is the object that stores the open date, close date, and issue link for that pull request. The reason
// why I said I think the pull request number is the same as the number of the issue it affects is because
// if you take any object from the map, say you do mapPRnumberToCloseOpenDateObjects.get(10). You'll
// get an object with a pull request number 10. Now if you take this object and look at it's "IssueLink"
// field, the very last part of the link will have the number 10, and if you look at the github API
// it says for a single issue, you do: /repos/:owner/:repo/issues/:issue_number <---- As you can see,
// the IssueLink field will have this structure and in place of the issue_number, the field will be 10
// for our example object.
mapPRnumberToCloseOpenDateObjects.set(listOfPrNumbers[n], temp);
}
//up to this point, we have the pull request numbers. we will now start getting the commits associated with
//each pull request
var j;
for (j = 0; j < listOfPrNumbers.length; j++){
var currentApiLink = "https://api.github.com/repos/elixir-lang/elixir/pulls/" + listOfPrNumbers[j] + "/commits";
const res = await axios.get(currentApiLink);
//here we map a single pull request to the information containing the commits. I'll just warn you in
// advance: there's another object called mapPRNumbersToCommitObjects. THIS MAP IS DIFFERENT! I know it's
// subtle, but I hope the language can make the distinction: mapPullRequestNumberToCommits will just
// map a pull request number to some data about the commits it's linked to. In contrast,
// mapPRNumbersToCommitObjects will be the map that actually maps pull request numbers to objects
// containing information about the commits a pull request is associated with!
mapPullRequestNumberToCommits.set(listOfPrNumbers[j], res['data']);
}
// console.log("hewoihoiewa");
}
async createCommitObjects(){
var x;
// the initial loop using x will loop over all pull requests and get the associated commits
for (x = 0; x < listOfPrObjects.length; x++){
//here we will get the commits
var currCommitObjects = mapPullRequestNumberToCommits.get(listOfPrObjects[x]['number']);
//console.log('dhsiu');
// the loop using y will iterate over all commits that we get from a single pull request
var y;
for (y = 0; y < currCommitObjects.length; y++){
var currentSHA = currCommitObjects[y]['sha'];
listOfSHAs.push(currentSHA);
var currApiLink = "https://api.github.com/repos/elixir-lang/elixir/commits/" + currentSHA;
const response = await axios.get(currApiLink);
//console.log("up to here");
// here we start extracting some information from a single commit
var currentAuthorName = response['data']['commit']['committer']['name'];
var currentDate = response['data']['commit']['committer']['date'];
var currentFiles = response['data']['files'];
// this loop will iterate over all changed files for a single commit. Remember, every commit has a list
// of changed files, so this loop will iterate over all those files, get the necessary information
// from those files.
var z;
// we create this temporary list of file objects because for every file, we want to make an object
// that will store the necessary information for that one file. after we store all the objects for
// each file, we will add this list of file objects as a field for our bigger commit object (see down below)
var tempListOfFileObjects = [];
for (z = 0; z < currentFiles.length; z++){
var fileInConsideration = currentFiles[z];
var nameOfFile = fileInConsideration['filename'];
var numberOfAdditions = fileInConsideration['additions'];
var numberOfDeletions = fileInConsideration['deletions'];
var totalNumberOfChangesToFile = fileInConsideration['changes'];
//console.log("with file");
var tempFileObject = {fileName: nameOfFile, totalAdditions: numberOfAdditions,
totalDeletions: numberOfDeletions, numberOfChanges: totalNumberOfChangesToFile};
// we add the same file objects to both a temporary, local list and a global set. Don't be tripped
// up by this; they're doing the same thing!
setOfFileObjects.add(tempFileObject);
tempListOfFileObjects.push(tempFileObject);
}
// here we make an object that stores information for a single commit. sha, authorName, date are single
// values, but files will be a list of file objects and these file objects will store further information
// for each file.
var tempObj = {sha: currentSHA, authorName: currentAuthorName, date: currentDate, files: tempListOfFileObjects};
var currPrNumber = listOfPrObjects[x]['number'];
console.log(currPrNumber);
// here we will make a single pull request number to an object that will contain all the information for
// every single commit associated with that pull request. So for every pull request, it will map to a list
// of objects where each object stores information about a commit associated with the pull request.
mapPRNumbersToCommitObjects.set(currPrNumber, tempObj);
}
}
return mapPRNumbersToCommitObjects;
}
startParsingPullRequests() {
this.getListOfPullRequests(gitPullApiLink + "?state=all").then(() => {
this.getCommitsForEachPullRequestAndPRinformation().then(() => {
this.createCommitObjects().then((response) => {
console.log("functions were successful");
return mapPRNumbersToCommitObjects;
}).catch((error) => {
console.log("printing first error");
// console.log(error);
})
}).catch((error2) => {
console.log("printing the second error");
console.log(error2);
})
}).catch((error3) => {
console.log("printing the third error");
// console.log(error3);
});
}
//adding some getter methods so they can be used to work with whatever information people may need.
//I start all of them with the this.startParsingPullRequests() method because by calling that method it gets all
// the information for the global variables.
async getSetOfFileObjects(){
var dummyMap = this.startParsingPullRequests();
return setOfFileObjects;
}
async OpenCloseDateObjects(){
var dummyMap = this.startParsingPullRequests();
return mapPRnumberToCloseOpenDateObjects;
}
async getNumbersOfTargetedIssues(){
var dummyMap = this.startParsingPullRequests();
return listOfNumbersOfTargetedIssues;
}
}
I then try to play around and run the function to make sure all the data I need is there by doing:
var dummy = new PullRequestParser();
var dummyMap = dummy.startParsingPullRequests();
And when I run it on webstorm using:
node PullRequestParser.js
It will print out some pull request numbers, then stop about halfway with a 403 error. I know what the 403 error is, but I'm wondering if there's anything on my end to stop it from happening, or is it just a matter of working with another repo that won't throw me this error. Thanks!
The 403 error from the server means that your access is forbidden. You either need to use different credentials (that is, log in as a user with that access), not use that repository, or gracefully handle the error and do something else. Retrying will not be effective, since GitHub wouldn't be very secure if it just let you have access to things you weren't supposed to.
I'm newbie in JS and I need to create simple CSV parser for test data. And I have a very strange problem.
This is code of my test:
'use strict';
const assert = require('assert');
const HomePage = require('../pages/HomePage');
const csv = require('../tools/CsvReader');
describe('Check if values are correct', () => {
let dataProvider = csv.readFromCsv("test/resources/places.csv");
function valuesTest(city, expectedLat, expectedLong) {
it('Data must match', () => {
let searchPlace = HomePage
.open()
.findByPlaceName(city)
.getSearchPlace();
assert.strictEqual(searchPlace.getLatitude(), expectedLat);
assert.strictEqual(searchPlace.getLongtitude(), expectedLong);
console.log(dataProvider[0].CITY);
});
}
for (let i = 0; i < dataProvider.length; i++) {
valuesTest(dataProvider[i].CITY, dataProvider[i].LAT, dataProvider[i].LONG)
}
});
And code of my CSV-reader:
'use strict';
const csv = require('csv-parser');
const fs = require('fs');
class CsvReader {
readFromCsv(path) {
let results = [];
fs.createReadStream(path)
.pipe(csv())
.on('data', (data) => results.push(data));
return results;
}
}
module.exports = new CsvReader();
And this is my CSV:
CITY,LAT,LONG
Kyiv,50.447731,30.542721
Lviv,49.839684,24.029716
Ivano-Frankivsk,48.922634,24.711117
The problem is as follows: Why can I use the variable "dataProvider" in the "valuesTest" block and it works correctly and returns the value of the CITY variable, but in the "for" loop I can't use it (variable "dataProvider", "CITY", etc. is unavailable there), although they are located in the same block "describe".
Your CSV Reader is an asynchronous operation. I suspect your for loop gets executed even before the csv is parsed and value is returned. Try putting your for loop in a function and passing to the readFromCsv function as a callback. Call this function on the data event, where you will be sure to get the data.
You should pass a callback to readFromCsv that will be executed when the createReadStream is complete. You can determine when createReadStream is complete by listening to its end event. (See csv-parser example code for instance).
In your case, you could do something like:
readFromCsv(path, onEnd) {
let results = [];
fs.createReadStream(path)
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => onEnd(results));
}
csv.readFromCsv("test/resources/places.csv", function onData(data) {
for (let i = 0; i < data.length; i++) {
valuesTest(data[i].CITY, data[i].LAT, data[i].LONG);
}
});
You should use the end event instead of the data event to determine when the stream is complete. If you returned after the first data event, you might not get all of the data.
The data event is fired once per data chunk. If your data has more than one "chunk", you'll truncate it if you return in the data callback. See nodeJS docs for details on the different event types. You might not notice a difference with small test files, but make a larger file and you'll see the difference.
I am trying to scrape data from a bricklet in the UI(i.e. HTML dataTable) and using a testCafe client function to do this but I haven't been successful. I have a few questions about my code and would like someone to point me in the right direction.
I first put my client function in the test file(test.js) which houses all my other test cases and called the function from one of my tests. Just like this example here: - https://devexpress.github.io/testcafe/documentation/test-api/obtaining-data-from-the-client/examples-of-using-client-functions.html check section "complex DOM queries" but testCafe gets stuck, the browser closes but the execution is stuck
Here is my client function. It is in my file that houses all my tests - test.js
fixture`Getting Started`
.page`${config.baseUrl}`;
const getTableRowValues = ClientFunction(() => {
console.log("inside client function");
const elements = document.querySelector('#bricklet_summary_studtable > tbody').querySelectorAll('tr td');
const array = [];
console.log(elements.length);
for (let i = 0; i <= elements.length; i++) {
console.log("inside for");
const customerName = elements[i].textContent;
array.push(customerName);
}
return array;
});
Here is my test case:
test('My 4th test - Check the bricklet data matches the expected data', async t => {
await t.navigateTo('https://myurl.com/app/home/students');
await page_studTest.click_studentlink();
await t
.expect(await page_studTest.exists_ListPageHeader()).ok('do something async', { allowUnawaitedPromise: true })//check the compare button does not exists
await t .navigateTo('https://myurl.com/app/home/students/application/stud/id/details/test.html')
await t
.expect(await page_studTest.getText_studHeader(t)).eql('student123',
"the header text does not match");
let arr = await getTableRowValues();
await console.log(arr);
});
I thought this will get the values from the UI in an array and I will compare it to another array of test values that I will hard code later.
At first, I tried client functions in my page class(page object model: https://devexpress.github.io/testcafe/documentation/recipes/use-page-model.html) and I put the client function in the constructor and called it from a async function in same page class and called the async function from my test.js. All my tests are structured this way but this only prints the following in the console
Valuesfunction __$$clientFunction$$() {
const testRun = builder._getTestRun();
const callsite = (0, _getCallsite.getCallsiteForMethod)(builder.callsiteNames.execution);
const args = [];
// OPTIMIZATION: don't leak `arguments` object.
for (let i = 0; i < arguments.length; i++) args.push(arguments[i]);
return builder._executeCommand(args, testRun, callsite);
}
which is not useful to debug the problem.
There are no examples on testCafe site as to how/where to put the client function when you use the page-object model. Could someone, please share some insight? I am interested in knowing the best way to structure my tests.
I didn't find any problems in your code which can make TestCafe hang. I didn't find any syntax errors or incorrect calls to TestCafe methods either. I only wish that you take note that the await keyword should not be called before console.log. Though this should not lead to any issues.
Probably the use of a custom promise with the { allowUnawaitedPromise: true } option can lead to problems, however it's difficult to determine it without the full project.
I recommend you prepare a simple project with a sample test file to demonstrate the issue and create a separate bug report in the TestCafe repository using the following form
So, finally I tried to return a promise from my client function and then it worked.
const getTableRowValues = ClientFunction(() => {
const array = [];
return new Promise(resolve => {
var elements = document.querySelectorAll('#bricklet_summary_studtable > tbody > tr > *');
elements.forEach(function (element, i) {
let text = element.textContent;
array[i] = text.trim();
});
resolve(array);
});
});
I resolve a single dimensional array as the assertion wasn't working with a 2D array in the test when I compare the result of the client function to a 2D expected value.However this serves the purpose for now.
I'm in need of some minor assistance. I'm having trouble getting an array (larray3) populated with two other array objects (larray1 and larray2) to pass both from data.js into the subsequent model.js and view.js. Data.js correctly builds the multidimensional array however when the results are received in model.js/view.js I only receive the results for larray1. Because only the first values come thru I cannot tell if both larray1 and larray2 are actually passing thru. Can someone please tell me how I should alter my syntax in either model.js or view.js to access both array values or what else I could change? Thank you in advance.
data.js.
function getCountries(done) {
var sqlite3 = require('sqlite3').verbose();
var file = 'db/locations.sqlite3';
var db = new sqlite3.Database(file);
var larray1 = [];
var larray2 = [];
var larray3 = [];
db.all('SELECT * FROM Country', function(err, rows) {
// This code only gets called when the database returns with a response.
rows.forEach(function(row) {
larray1.push(row.CountryName);
larray2.push(row.CountryCode);
})
larray3.push(larray1);
larray3.push(larray2);
return done(larray3[0], larray3[1]);
});
db.close();
}
model.js
function Countries(done) {
//Pull location values from data
return getCountries(done);
}
view.js
function viewCountries() {
var viewCou = Countries(function(results) {
// Code only gets triggered when Countries() calls return done(...);
var container = document.getElementById('country-select');
var fragment = document.createDocumentFragment();
results.forEach(function(loc, index) {
var opt = document.createElement('option');
opt.innerHTML = loc;
opt.value = loc;
fragment.appendChild(opt);
});
container.appendChild(fragment);
})
}
In data.js you send two arguments to the done callback:
return done(larray3[0], larray3[1]);
This done function is passed through in your model.js:
return getCountries(done);
And that done is passed in from view.js:
Countries(function(results) { // ...
So it is this anonymous function (function(results) {...}) that is called in data.js. But notice that this function only has one parameter, so you're doing nothing with the second argument that data.js sends. result gets the value of larray3[0], but larray3[1] is not captured anywhere.
You could solve this in different ways. Personally, I think the design with two arrays is wrong from the start. I would not separate data that belongs in pairs (name and code) into two different arrays.
Instead make an array of objects, and pass that single array around:
In data.js:
rows.forEach(function(row) {
larray1.push({
name: row.CountryName,
code: row.CountryCode
});
})
return done(larray1);
In view.js:
opt.textContent = loc.name;
opt.value = loc.code;
Side-note: .textContent is preferred over .innerHTML when assigning plain text.