I'm using Koa.js with Mongoose.js. There is a collection in my mongo named css. Which have following schema:
_id
css_name
css_value
I have an array containing a huge list like:
var list = ['font-color', 'back-color', 'font-family', 'back-image', 'back-repeat', ... ];
Now I've declared a mongoose model named css and executing a loop like this:
for(var i = 0; i < list.length; i++) {
console.log(yield css.findOne({css_name: list[i]}).exec());
}
If I execute the code above it gives null in the console. Whenever I omit the loop, it works perfectly:
//for(var i = 0; i < list.length; i++) {
console.log(yield css.findOne({css_name: 'font-color'}).exec());
//}
So the problem is with the loop. Can anyone suggest a better working way to fetch all the values from a loop using the mongoose model?
I don't really see a problem with your code to be honest, it should work, and if you see null as result is because the query didn't return any document, try mongoose.set('debug', true) and execute the queries yourself. Anyway, if you just need all results at once, just use co-each as follow:
var each = require('co-each')
var styles = ['font-color', 'back-color', 'font-family', 'back-image', 'back-repeat']
// executed in parallel
var results = yield each(styles, function *getStyle(style) {
return yield css.findOne({ css_name: style }).exec()
})
console.log(results)
Related
So i'm attempting to write a google parser.
The idea of my tool is it takes search queries and searches google for them and returns URLs. It is working good so far but now im trying to set a page configuration and im having troubles, my code is:
const needle = require("needle") //for making get request
const sp = require("serp-parser") //for parsing data from the request
const queryup = "watch movies online free" //my search data
const query = encodeURI(queryup) //my search data so google can read it
var page = 0; //initializing the page counter
let pages = 5; //setting amount of pages to loop through
for (var i = 0; i < pages; i++) { //my loop
needle.get(`https://www.google.com/search?q=${query}&start=${page}`, function(err, response){ //MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
page += 10 //adding to page value (every 10 page value is 1 extra page)
console.log(`----- Page number: `+ page / 10+" -----") //logging the number of the page to confirm that it is indeed increasing the page value
let results = response.body; //defining the body of my request
parser = new sp.GoogleNojsSERP(results); //initializing the parser
let parsed = parser.serp //parsing the body
let objarray = parsed.organic; //parsed body (returns as an array of json objects)
for (var i = 0; i < objarray.length; i++) { //loop the logging of each url
let url = objarray[i].url //defining url
console.log(url) //logging each url
}
});
}
without a billion comments:
const needle = require("needle")
const sp = require("serp-parser")
const queryup = "watch movies online free"
const query = encodeURI(queryup)
var page = 0;
let pages = 5;
for (var i = 0; i < pages; i++) {
needle.get(`https://www.google.com/search?q=${query}&start=${page}`, function(err, response){
//^^^^^ MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
page += 10
console.log(`----- Page number: `+ page / 10+" -----")
let results = response.body;
parser = new sp.GoogleNojsSERP(results);
let parsed = parser.serp
let objarray = parsed.organic;
for (var i = 0; i < objarray.length; i++) {
let url = objarray[i].url
console.log(url)
}
});
}
This seems to be an issue with async.
I'm not familiar with needle, but I know that external queries are basically never synchronous.
The problem you're experiencing is basically, the actual web query is happening after the loop first runs and has already incremented page to 50. Then, 5 queries are constructed, each one with page=50, because async is complicated and difficult to manage.
Under the hood, the engine is essentially doing literally everything else it can possibly do first, and THEN doing your web queries.
A trip through the needle npm docs tells me that you can use alternative syntax to get needle to return a promise instead, which can then be wrapped in an asynchronous function and managed through await to force synchronous behavior, which is what you're after:
const needle = require('needle');
const sp = require('serp-parser');
const queryup = 'watch movies online free';
const query = encodeURI(queryup);
let page = 0;
const pages = 5;
const googler = async function () {
for (let i = 0; i < pages; i++) {
try {
const response = await needle('get', `https://www.google.com/search?q=${query}&start=${page}`);// MY MAIN PROBLEM <<<--- The issue is its adding to the page value but its not effecting it here, why?
console.log('----- Page number: ' + page / 10 + ' -----');
const results = await response.body;
const parser = new sp.GoogleNojsSERP(results);
const parsed = parser.serp;
const objarray = parsed.organic;
for (let i = 0; i < objarray.length; i++) {
const url = objarray[i].url;
console.log(url);
}
} catch (err) {
console.error(err);
}
page += 10;
}
};
googler();
The key differences:
Per the needle docs, rather than the request method being a method on the needle object, it's instead the first argument you pass directly to invoking needle itself as a function.
When you manage promises with await, a rejected promise throws an error that should be caught with a traditional try/catch block; I've done that here. Though, if needle is anything like node-fetch it probably basically never throws errors, but it's good practice.
One of my extensions automatically changed your var declarations to let and not-reassigned let declarations to const; you're welcome to change them back.
This is a classic asynchronous problem. Add another console.log() immediately before the needle.get() call (and after the for statement) and you will see what is going wrong: All of the needle.get() calls execute before any of the callbacks where you do the page += 10. Then, after the for loop completes, all of the callbacks are executed. But it is too late for this to have any effect on the start= parameter.
One way to fix this could be to move the body of this for loop (the needle.get() and its callback) into a separate function. Initialize your variables and call this function once. Then at the end of the callback, do your page += 10 and update any other variables you need to, and call this function again from there if there are more pages left that you want to load. If you have completed all of the pages, then don't make that call. The for loop is not needed with this technique.
Or, you could keep your current code but move the page += 10 after the callback but still inside the outer for loop. That way this variable will be incremented as you expect. I don't necessarily recommend this, as Google may get unhappy about receiving the get requests so rapidly and may start blocking your calls or throwing CAPTCHAs at you.
There may be an issue of whether this kind of scraping is allowed by Google's Terms of Service, but I will leave that question to you and your legal advisors.
Also, I would avoid using var anywhere. Use const or let instead, and prefer const over let except when you need to reassign the variable.
One tip: in most cases where you use a numeric for loop to iterate over an array, the code will be cleaner if you use a for..of loop. For example, this bit of code:
let parsed = parser.serp
let objarray = parsed.organic;
for (var i = 0; i < objarray.length; i++) {
let url = objarray[i].url
console.log(url)
}
could be more simply written as:
for (const result of parser.serp.organic) {
console.log(result.url)
}
(I know that is just a bit of debug code, but this is a good habit to get into.)
Finally, watch your indentation and be sure to indent nested blocks or functions. I took the liberty of adding some indentation for you.
I am trying to scrape data from a bricklet in the UI(i.e. HTML dataTable) and using a testCafe client function to do this but I haven't been successful. I have a few questions about my code and would like someone to point me in the right direction.
I first put my client function in the test file(test.js) which houses all my other test cases and called the function from one of my tests. Just like this example here: - https://devexpress.github.io/testcafe/documentation/test-api/obtaining-data-from-the-client/examples-of-using-client-functions.html check section "complex DOM queries" but testCafe gets stuck, the browser closes but the execution is stuck
Here is my client function. It is in my file that houses all my tests - test.js
fixture`Getting Started`
.page`${config.baseUrl}`;
const getTableRowValues = ClientFunction(() => {
console.log("inside client function");
const elements = document.querySelector('#bricklet_summary_studtable > tbody').querySelectorAll('tr td');
const array = [];
console.log(elements.length);
for (let i = 0; i <= elements.length; i++) {
console.log("inside for");
const customerName = elements[i].textContent;
array.push(customerName);
}
return array;
});
Here is my test case:
test('My 4th test - Check the bricklet data matches the expected data', async t => {
await t.navigateTo('https://myurl.com/app/home/students');
await page_studTest.click_studentlink();
await t
.expect(await page_studTest.exists_ListPageHeader()).ok('do something async', { allowUnawaitedPromise: true })//check the compare button does not exists
await t .navigateTo('https://myurl.com/app/home/students/application/stud/id/details/test.html')
await t
.expect(await page_studTest.getText_studHeader(t)).eql('student123',
"the header text does not match");
let arr = await getTableRowValues();
await console.log(arr);
});
I thought this will get the values from the UI in an array and I will compare it to another array of test values that I will hard code later.
At first, I tried client functions in my page class(page object model: https://devexpress.github.io/testcafe/documentation/recipes/use-page-model.html) and I put the client function in the constructor and called it from a async function in same page class and called the async function from my test.js. All my tests are structured this way but this only prints the following in the console
Valuesfunction __$$clientFunction$$() {
const testRun = builder._getTestRun();
const callsite = (0, _getCallsite.getCallsiteForMethod)(builder.callsiteNames.execution);
const args = [];
// OPTIMIZATION: don't leak `arguments` object.
for (let i = 0; i < arguments.length; i++) args.push(arguments[i]);
return builder._executeCommand(args, testRun, callsite);
}
which is not useful to debug the problem.
There are no examples on testCafe site as to how/where to put the client function when you use the page-object model. Could someone, please share some insight? I am interested in knowing the best way to structure my tests.
I didn't find any problems in your code which can make TestCafe hang. I didn't find any syntax errors or incorrect calls to TestCafe methods either. I only wish that you take note that the await keyword should not be called before console.log. Though this should not lead to any issues.
Probably the use of a custom promise with the { allowUnawaitedPromise: true } option can lead to problems, however it's difficult to determine it without the full project.
I recommend you prepare a simple project with a sample test file to demonstrate the issue and create a separate bug report in the TestCafe repository using the following form
So, finally I tried to return a promise from my client function and then it worked.
const getTableRowValues = ClientFunction(() => {
const array = [];
return new Promise(resolve => {
var elements = document.querySelectorAll('#bricklet_summary_studtable > tbody > tr > *');
elements.forEach(function (element, i) {
let text = element.textContent;
array[i] = text.trim();
});
resolve(array);
});
});
I resolve a single dimensional array as the assertion wasn't working with a 2D array in the test when I compare the result of the client function to a 2D expected value.However this serves the purpose for now.
I was wondering what the issue with the bottom loop is or if I'm going through the last json wrong somehow when I'm trying to log it into the console. The arrays are above the given code and the first two loops work fine. I'm trying to return goals but the reality is I want to find an efficient way to return all of the stats.
d3.json('https://statsapi.web.nhl.com/api/v1/teams', function(data) {
for (i=0; i < 31; i++) {
teamID.push(data.teams[i].id);
}
});
console.log(teamID);
// request roster json data from API and loop through roster to get all player IDS
// and append them to the playerList array
d3.json('https://statsapi.web.nhl.com/api/v1/teams/1/?expand=team.roster', function(data) {
for (i=0; i < data.teams[0].roster.roster.length; i++) {
playerList.push(data.teams[0].roster.roster[i].person.id);
}
});
console.log(playerList);
// request player stat json data from API and loop through all players to get all stat
// and append them to an array
var playerStats = [];
for (i = 0; i < playerList.length; i++) {
d3.json('https://statsapi.web.nhl.com/api/v1/people/' + playerList[i] + '/stats/?stats=statsSingleSeason&season=20172018', function(data) {
console.log(data.stats[0].splits[0].stat.goals);
});
// console.log(playerStats);
};
Your final loop is probably attempting to initialize / run at the same time as the HTTP calls are being returned from the APIs. Since you are using callbacks to get the details, rather than promises, then you will need to do this in callback form. Here is the best I can do without you actually showing me the full code:
d3.json('https://statsapi.web.nhl.com/api/v1/teams', function(teamResponse) {
var teamIds = teamResponse.teams.filter((team, i) => i < 31)
.map((team) => team.id);
// I use the functional approach above because I think it is cleaner than loops.
// for (i=0; i < 31; i++) {
// teamID.push(data.teams[i].id);
//}
d3.json('https://statsapi.web.nhl.com/api/v1/teams/1/?expand=team.roster', function(rosterResponse) {
var playerIdList = rosterResponse.teams[0].roster.roster
.map((roster) => roster.person.id);
// Swap this out for the functional method above.
//for (i=0; i < data.teams[0].roster.roster.length; i++) {
// playerList.push(data.teams[0].roster.roster[i].person.id);
//}
for(var i = 0; i < playerIdList; i++) {
d3.json('https://statsapi.web.nhl.com/api/v1/people/' + playerIdList[i] + '/stats/?stats=statsSingleSeason&season=20172018', function(data) {
console.log(data.stats[0].splits[0].stat.goals);
});
}
});
});
Promises (Promise.all) are not supported at all in Internet Explorer (they are in Edge) and some older versions of other browsers. Arrow functions are also not supported in these browsers.
I assume that when you need to support older browsers you can use babel (with webpack) or know how to write ES5.
d3.json returns a promise so you can leave out the callback and uses promises:
Promise.all([
d3.json('https://statsapi.web.nhl.com/api/v1/teams'),
d3.json('https://statsapi.web.nhl.com/api/v1/teams/1/?expand=team.roster')
])
.then(
([teams,playerData])=>{
const playerList = playerData.teams[0].roster.roster.map(
player=>player.id
);
return Promise.all(
playerList.map(
playerID=>
d3.json(`https://statsapi.web.nhl.com/api/v1/people/${playerID}/stats/?stats=statsSingleSeason&season=20172018`)
)
).then(
(playerStats)=>[teams,playerData,playerStats]
)
}
)
.then(
([teams,playerData,playerStats])=>{
console.log("teams:",teams);
console.log("playerData:",playerData);
console.log("playerStats:",playerStats);
}
)
.catch(
err=>console.warn("Something went wrong:",err)
);
I did not comment on how the code works so please let me know if you have specific questions about the code. I suggest reading this if you don't know why promises are used. And google "mdn promise all" before asking what promise all does.
I am using Meteor, which uses Mongodb as its database. I have code that inserts several documents into a collection when users fill out a form. When these documents are inserted, I would like to fire some JavaScript code within the server side directories that sorts through the collection in question for documents with matching fields as the documents just inserted.
My problem is that I do not know how to fire code on the server when the new documents arrive. Would it make sense to Meteor.call a Meteor.method at the end of the code involved with inserting, with the Meteor.method called preforming the sorting code I need?
Edit:
As you can see, in the below code I'm not calling any Meteor methods as none exist yet. The vast majority of this code is simply lead up for the insert({}) at the end of the page, so I think it can be safely ignored. The only server side code I have is to declare the possibleGames mongo collection.
I am not sure what you mean by call a plain JavaScript function, my problem is getting any code firing at all.
possibleGames = new Mongo.Collection("possibleGames");
Template.meet_form.events({
"submit .meet_form": function(event, template){
event.preventDefault();
var user = Meteor.userId();
var where = event.target.where.value;
var checkedGames = [];
function gameCheck (game) {
if (game.checked === true){
checkedGames.push(game.value);
};
};
var checkedDays = [];
function dayCheck (day) {
if (day.checked === true){
checkedDays.push(day.value);
};
};
console.log(event.target.where.value)
gameCheck(event.target.dnd);
gameCheck(event.target.savageWorlds);
gameCheck(event.target.shadowRun);
console.log(checkedGames);
dayCheck(event.target.monday);
dayCheck(event.target.tuesday);
dayCheck(event.target.wednesday);
dayCheck(event.target.thursday);
dayCheck(event.target.friday);
dayCheck(event.target.saturday);
dayCheck(event.target.sunday);
console.log(checkedDays);
var whereWhat = [];
for (i = 0; i < checkedGames.length; i++) {
var prepareWhereWhat = where.concat(checkedGames[i]);
whereWhat.push(prepareWhereWhat);
};
console.log(whereWhat);
var whereWhatWhen = [];
for (a = 0; a < whereWhat.length; a++) {
var prepareWWW1 = whereWhat[a];
for (b = 0; b < checkedDays.length; b++) {
var prepareWWW2 = prepareWWW1.concat(checkedDays[b]);
whereWhatWhen.push(prepareWWW2);
};
};
console.log(whereWhatWhen);
for (i = 0; i < whereWhatWhen.length; i++) {
possibleGames.insert({
game: whereWhatWhen[i],
user: user,
created_on: new Date().getTime()
})
}
}
});
You don't need to do a meteor.call on the server because you're already on the server.
Just call a plain javascript function.
If what you want to call from your first Meteor.method is already in another Meteor.method, then refactor that function to extract out the common bit.
Some code would also help if this is still confusing.
Here is a simple task I would like to accomplish on Parse.com with Cloud Code.
The task consists to delete a Unit and what is related to it.
One Unit has several Sentences related to it and each Sentence has one or more Translations.
So when the task is performed, the Unit as well as the Sentence and Translations should be deleted.
I have a strong feeling I should be using Promises (and chain them up) in order to do what I want in a good manner.
Below is the code I wrote, but it works only partially (The translations are deleted, not the rest).
Parse.Cloud.define("deleteUnitAndDependencies", function(request, response) {
var unitListQuery = new Parse.Query("UnitList");
unitListQuery.equalTo("objectId", request.params.unitID);
unitListQuery.equalTo("ownerID", request.params.userID);
unitListQuery.find().then(function(resUnit) {
var sentenceListQuery = new Parse.Query("SentenceList");
sentenceListQuery.equalTo("unit", resUnit[0]);
return sentenceListQuery.find();
}).then(function(resSentence) {
var translatListQuery = new Parse.Query("TranslatList");
for (i = 0; i < resSentence.length; i++) {
var query = new Parse.Query("TranslatList");
query.equalTo("sentence", resSentence[i]);
translatListQuery = Parse.Query.or(translatListQuery, query);
}
return translatListQuery.find();
}).then(function(resTranslat) {
for (iT = 0; iT < resTranslat.length; iT++) {
resTranslat[iT].destroy({});
}
});
});
I surely need to add some lines of code like:
resSentence[x].destroy({});
and:
resUnit[0].destroy({});
The problem is that I do not quite see where is the adequate place for that.
Collect the objects to be deleted then use Parse.Object.destroyAll(someArray); to delete all at once.
In cases like this I like to use a scope variable to hold things for later use.
var scope = {
sentences: [],
units: []
};
// later inside then block...
scope.sentences.push(resSentence[i]);
// ...now we have them collected safely
.then(function() {
return Parse.Object.destroyAll(scope.sentences);
})