When I check request data in Postman it returns HTML file and contains JavaScript value.
<html>
<body>
HTML VALUES...
</body>
<script>
var _ITEMID = '29041549';
var ITEM_VID = '612c86e6c6f7840001d0c821';
var _SOLDOUT = '1' == 0;
var _DISCOUNTPRICE = '110.00';
var _SIZE = 'S,M,L,XL';
var _ITEMTYPE = '0';
var _COLORPICS = '';
</script>
<script>another script value</script>
<script>another script value</script>
<script>another script value</script>
<script>another script value</script>
</html>
I use got package to request data.
And I want to get script's value, but when I tried this code, it returns undefined.
I already checked this article and modify this codes but it seems it doesn't fit my case.
const got = require('got');
const cheerio = require('cheerio');
const data = await got(`https://www.vvic.com/item/${itemDetailURL}.html`);
// console.log(data);
const $ = cheerio.load(data);
const calData = $('script').get()[0];
console.log(calData); // Returns undefined
[console.log($('script'));]
LoadedCheerio {
length: 0,
options: { xml: false, decodeEntities: true },
_root: <ref *1> LoadedCheerio {
'0': Document {
type: 'root',
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array]
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *1]
},
prevObject: <ref *1> LoadedCheerio {
'0': Document {
type: 'root',
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array]
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *1]
}
}
[console.log($('script').get());]
[] // Returns empty array
Related
I scrape with cheerio and node-fetch a XML data.
I would like my scraping object in JSON or array convert but how make I this?
How I can convert and access this in a array?
const $ = load(responseText, {
xmlMode: true,
});
let dataLength = ($('item').length);
console.log(dataLength);
const items = ($('item'));
My object is:
prevObject: <ref *7> LoadedCheerio {
'0': Document {
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array],
type: 'root'
},
length: 1,
options: { xml: false, decodeEntities: true, xmlMode: true },
_root: [Circular *7]
}
I'm writing a Node.JS code which acts as an intermediate between JavaScript client and Google DialogFlow. My problem is, when I try to echo the JSON object response to the client through a web socket connection, the nested values always read as null.
My JSON Object:
{
responseId: 'xxxx-xxxxxx-xxxxxx-xxxx',
recognitionResult: null,
queryResult: {
fulfillmentMessages: [ [Object] ],
outputContexts: [ [Object], [Object], [Object] ],
queryText: '21',
speechRecognitionConfidence: 0,
action: 'age',
parameters: { fields: [Object] },
allRequiredParamsPresent: true,
fulfillmentText: 'All right!',
webhookSource: '',
webhookPayload: null,
intent: {
inputContextNames: [],
events: [],
trainingPhrases: [],
outputContexts: [],
parameters: [],
messages: [],
defaultResponsePlatforms: [],
followupIntentInfo: [],
name: 'Intent name',
displayName: 'Default Intent',
priority: 0,
isFallback: false,
webhookState: 'WEBHOOK_STATE_UNSPECIFIED',
action: '',
resetContexts: false,
rootFollowupIntentName: '',
parentFollowupIntentName: '',
mlDisabled: false
},
intentDetectionConfidence: 1,
diagnosticInfo: null,
languageCode: 'en',
sentimentAnalysisResult: null
},
webhookStatus: { details: [], code: 0, message: '' },
outputAudio: <Buffer >,
outputAudioConfig: {
audioEncoding: 'OUTPUT_AUDIO_ENCODING_OGG_OPUS',
sampleRateHertz: 0,
synthesizeSpeechConfig: null
}
}
I'm trying to access the name value of intent. I also need to pass all the other values as well.
My Web Socket code:
const ws = require('ws');
const wss = new ws.Server({ noServer: true });
function onConnect(ws) {
ws.on('message', function (newMessage) {
ws.send(JSON.stringify(data));
}
}
(I've trimmed out parts that are irrelevant)
I'm new to node.js so apologies if this is something very simple.
I have the below node js script:
var http = require("http");
var bodyParser = require("body-parser");
var options = {
"method" : "GET",
"hostname" : "xxx.xxx.xxx.xxx",
"port" : "18080",
"path" : "/api/v1/applications/app-20180103124606-0007/stages/0"
};
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = JSON.parse(Buffer.concat(chunks));
console.log(body);
});
});
req.end();
It returns this JSON:
[ { status: 'COMPLETE',
stageId: 0,
attemptId: 0,
numActiveTasks: 0,
numCompleteTasks: 1,
numFailedTasks: 0,
executorRunTime: 2738,
executorCpuTime: 1207164005,
submissionTime: '2018-01-03T12:46:10.796GMT',
firstTaskLaunchedTime: '2018-01-03T12:46:10.810GMT',
completionTime: '2018-01-03T12:46:14.513GMT',
inputBytes: 0,
inputRecords: 99171,
outputBytes: 0,
outputRecords: 0,
shuffleReadBytes: 0,
shuffleReadRecords: 0,
shuffleWriteBytes: 1468516,
shuffleWriteRecords: 3872,
memoryBytesSpilled: 0,
diskBytesSpilled: 0,
name: 'reduceByKey at /scripts/wordcount.py:37',
details: 'org.apache.spark.rdd.RDD.<init>(RDD.scala:104)\norg.apache.spark.api.python.PairwiseRDD.<init>(PythonRDD.scala:391)\nsun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\nsun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\nsun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\njava.lang.reflect.Constructor.newInstance(Constructor.java:423)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\npy4j.Gateway.invoke(Gateway.java:236)\npy4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)\npy4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)\npy4j.GatewayConnection.run(GatewayConnection.java:214)\njava.lang.Thread.run(Thread.java:748)',
schedulingPool: 'default',
accumulatorUpdates:
[ [Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object],
[Object] ],
tasks: { '0': [Object] },
executorSummary: { '0': [Object] } } ]
I now need to extract launchTime and taskTime. How is this done? I had previousy managed to extract data from JSON but I think I'm having trouble here as it's contained in an array.
You can use Array.prototype.forEach() to iterate over each member of the response body.
res.on('end', () => {
let body = JSON.parse(Buffer.concat(chunks))
body.forEach(item => {
// Do something with item
console.log(item)
})
})
If the array length is always 1, then you can access the first array element with body[0].
body[0].executorRunTime
body[0].firstTaskLaunchedTime
Otherwise you could iterate over the result with body.forEach().
I'm writing a Firebase Cloud Function and I'm trying to figure out how Promise.all works. In my code, I pass in an array of database queries and I'm trying the read the resulting array, but I'm only getting garbage:
T {
A:
P {
k: Sb { Ka: [Function: vb], ba: [Object] },
aa: P { k: [Object], aa: null, wb: [Object], Bb: '' },
wb: Zc { ld: [Object], ac: [Object] },
Bb: null },
V:
R {
u:
Gd {
app: [Object],
L: [Object],
Ua: [Object],
Sc: null,
ca: [Object],
td: 1,
Qa: [Object],
va: [Object],
qg: [Object],
jc: [Object],
ee: [Object],
md: [Object],
ia: [Object],
Xa: [Object],
cd: 2,
fe: null,
K: [Object] },
path: J { o: [Object], Y: 0 },
m:
Df {
xa: false,
ka: false,
Ib: false,
na: false,
Pb: false,
oa: 0,
kb: '',
bc: null,
xb: '',
Zb: null,
vb: '',
g: Tc {} },
Kc: false,
then: undefined,
catch: undefined },
g: Tc {} }
I'm expecting a simple json:
{
"name": "Foo",
"number": 2521
// And a few other fields
}
BTW, I watched Jen's video so I know what I'm doing is wrong anyway; I just want to know why my existing code doesn't work. (I haven't tested it, but I believe the solution is to return the raw snapshots in my db query and then do the .val() call.)
Relevant code if the links disappear:
function mergeTeams(duplicates) {
return Promise.all([
admin.database().ref(someRef).once('value', (snap) => {
return snap.val();
}),
admin.database().ref(someRef2).once('value', (snap) => {
return snap.val();
})
]).then(values => {
console.log(values);
const team1 = values[0];
const team2 = values[1];
console.log(team1);
console.log(team2);
}
So, here's the code that works (and the explanation below):
return Promise.all([
admin.database().ref(teamRef + duplicates.teamKey1).once('value'),
admin.database().ref(teamRef + duplicates.teamKey2).once('value')
]).then(values => {
const team1 = values[0].val();
const team2 = values[1].val();
console.log(team1);
console.log(team2);
});
The reason it works is because I've always getting the promises in the values array even though I didn't know it. Here's what Promise.all returns: an array with the raw result of the promises passed in. When I was returning stuff inside the success callback, that didn't actually do anything because it wasn't part of the promise; I was just returning random stuff to an empty void. And when I was printing the teams, I was actually logging the Firebase Snapshot object instead of the .val().
I am new to Node.js and MongoDB and I am really struggling to wrap my head around callbacks. I have read a few articles but it is still quite confusing to me. In the code below, I am trying to return the count of orders that have some properties which I have expressed in the query in orderModel.count(query, next):
controllers/order.js:
var mongoose = require ('../config/db');
var orderModel = require('../models/order').model;
var User = require('./user');
var Error = require('../config/error');
createOrder: function (user, order, next) {
if (newOrder.totalPrice > user.credit && orderModel.
count({$and: [{user: order.user}, {active: true}, {$or: [{status: 0}, {status: 1}]}]},
function(err, count){
if(err)
console.log(err);
else
count; }) > 0)
return next({error: Error.InsufficientCredits});
}
I don't think I am correctly obtaining the variable count because when I tried printing out the result of the second condition in the if statement, I got this data printed out:
Query {
_mongooseOptions: {},
mongooseCollection:
NativeCollection {
collection: Collection { s: [Object] },
opts: { bufferCommands: true, capped: false },
name: 'orders',
collectionName: 'orders',
conn:
NativeConnection {
base: [Object],
collections: [Object],
models: [Object],
config: [Object],
replica: false,
hosts: null,
host: 'dsXXXXXX.mlab.com',
port: XXXXXX,
user: 'XXXX',
pass: 'XXXX',
name: 'X',
options: [Object],
otherDbs: [],
_readyState: 1,
_closeCalled: false,
_hasOpened: true,
_listening: false,
db: [Object],
_events: {},
_eventsCount: 0 },
queue: [],
buffer: false,
emitter:
EventEmitter {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined } },
....
You need to put your logic inside the Model.count() method callback function as:
var mongoose = require ('../config/db'),
orderModel = require('../models/order').model,
User = require('./user'),
Error = require('../config/error');
var createOrder = function (user, order, next) {
orderModel.count({
"user": order.user,
"active": true,
"status": { "$in": [0, 1] }
}, function(err, count) { //<-- put logic in this callback
if (err) {
console.log(err);
throw err;
} else if (newOrder.totalPrice > user.credit && count > 0) {
// logic for creating order here
} else {
return next({ "error": Error.InsufficientCredits });
}
})
}