Cheerio XML to array or object - javascript

I scrape with cheerio and node-fetch a XML data.
I would like my scraping object in JSON or array convert but how make I this?
How I can convert and access this in a array?
const $ = load(responseText, {
xmlMode: true,
});
let dataLength = ($('item').length);
console.log(dataLength);
const items = ($('item'));
My object is:
prevObject: <ref *7> LoadedCheerio {
'0': Document {
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array],
type: 'root'
},
length: 1,
options: { xml: false, decodeEntities: true, xmlMode: true },
_root: [Circular *7]
}

Related

How to get javascript variable with Cheerio

When I check request data in Postman it returns HTML file and contains JavaScript value.
<html>
<body>
HTML VALUES...
</body>
<script>
var _ITEMID = '29041549';
var ITEM_VID = '612c86e6c6f7840001d0c821';
var _SOLDOUT = '1' == 0;
var _DISCOUNTPRICE = '110.00';
var _SIZE = 'S,M,L,XL';
var _ITEMTYPE = '0';
var _COLORPICS = '';
</script>
<script>another script value</script>
<script>another script value</script>
<script>another script value</script>
<script>another script value</script>
</html>
I use got package to request data.
And I want to get script's value, but when I tried this code, it returns undefined.
I already checked this article and modify this codes but it seems it doesn't fit my case.
const got = require('got');
const cheerio = require('cheerio');
const data = await got(`https://www.vvic.com/item/${itemDetailURL}.html`);
// console.log(data);
const $ = cheerio.load(data);
const calData = $('script').get()[0];
console.log(calData); // Returns undefined
[console.log($('script'));]
LoadedCheerio {
length: 0,
options: { xml: false, decodeEntities: true },
_root: <ref *1> LoadedCheerio {
'0': Document {
type: 'root',
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array]
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *1]
},
prevObject: <ref *1> LoadedCheerio {
'0': Document {
type: 'root',
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array]
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *1]
}
}
[console.log($('script').get());]
[] // Returns empty array

Not able to access nested JSON values after sending through socket

I'm writing a Node.JS code which acts as an intermediate between JavaScript client and Google DialogFlow. My problem is, when I try to echo the JSON object response to the client through a web socket connection, the nested values always read as null.
My JSON Object:
{
responseId: 'xxxx-xxxxxx-xxxxxx-xxxx',
recognitionResult: null,
queryResult: {
fulfillmentMessages: [ [Object] ],
outputContexts: [ [Object], [Object], [Object] ],
queryText: '21',
speechRecognitionConfidence: 0,
action: 'age',
parameters: { fields: [Object] },
allRequiredParamsPresent: true,
fulfillmentText: 'All right!',
webhookSource: '',
webhookPayload: null,
intent: {
inputContextNames: [],
events: [],
trainingPhrases: [],
outputContexts: [],
parameters: [],
messages: [],
defaultResponsePlatforms: [],
followupIntentInfo: [],
name: 'Intent name',
displayName: 'Default Intent',
priority: 0,
isFallback: false,
webhookState: 'WEBHOOK_STATE_UNSPECIFIED',
action: '',
resetContexts: false,
rootFollowupIntentName: '',
parentFollowupIntentName: '',
mlDisabled: false
},
intentDetectionConfidence: 1,
diagnosticInfo: null,
languageCode: 'en',
sentimentAnalysisResult: null
},
webhookStatus: { details: [], code: 0, message: '' },
outputAudio: <Buffer >,
outputAudioConfig: {
audioEncoding: 'OUTPUT_AUDIO_ENCODING_OGG_OPUS',
sampleRateHertz: 0,
synthesizeSpeechConfig: null
}
}
I'm trying to access the name value of intent. I also need to pass all the other values as well.
My Web Socket code:
const ws = require('ws');
const wss = new ws.Server({ noServer: true });
function onConnect(ws) {
ws.on('message', function (newMessage) {
ws.send(JSON.stringify(data));
}
}
(I've trimmed out parts that are irrelevant)

Why does returning snapshot.val() in a Promise when using Promise.all not work?

I'm writing a Firebase Cloud Function and I'm trying to figure out how Promise.all works. In my code, I pass in an array of database queries and I'm trying the read the resulting array, but I'm only getting garbage:
T {
A:
P {
k: Sb { Ka: [Function: vb], ba: [Object] },
aa: P { k: [Object], aa: null, wb: [Object], Bb: '' },
wb: Zc { ld: [Object], ac: [Object] },
Bb: null },
V:
R {
u:
Gd {
app: [Object],
L: [Object],
Ua: [Object],
Sc: null,
ca: [Object],
td: 1,
Qa: [Object],
va: [Object],
qg: [Object],
jc: [Object],
ee: [Object],
md: [Object],
ia: [Object],
Xa: [Object],
cd: 2,
fe: null,
K: [Object] },
path: J { o: [Object], Y: 0 },
m:
Df {
xa: false,
ka: false,
Ib: false,
na: false,
Pb: false,
oa: 0,
kb: '',
bc: null,
xb: '',
Zb: null,
vb: '',
g: Tc {} },
Kc: false,
then: undefined,
catch: undefined },
g: Tc {} }
I'm expecting a simple json:
{
"name": "Foo",
"number": 2521
// And a few other fields
}
BTW, I watched Jen's video so I know what I'm doing is wrong anyway; I just want to know why my existing code doesn't work. (I haven't tested it, but I believe the solution is to return the raw snapshots in my db query and then do the .val() call.)
Relevant code if the links disappear:
function mergeTeams(duplicates) {
return Promise.all([
admin.database().ref(someRef).once('value', (snap) => {
return snap.val();
}),
admin.database().ref(someRef2).once('value', (snap) => {
return snap.val();
})
]).then(values => {
console.log(values);
const team1 = values[0];
const team2 = values[1];
console.log(team1);
console.log(team2);
}
So, here's the code that works (and the explanation below):
return Promise.all([
admin.database().ref(teamRef + duplicates.teamKey1).once('value'),
admin.database().ref(teamRef + duplicates.teamKey2).once('value')
]).then(values => {
const team1 = values[0].val();
const team2 = values[1].val();
console.log(team1);
console.log(team2);
});
The reason it works is because I've always getting the promises in the values array even though I didn't know it. Here's what Promise.all returns: an array with the raw result of the promises passed in. When I was returning stuff inside the success callback, that didn't actually do anything because it wasn't part of the promise; I was just returning random stuff to an empty void. And when I was printing the teams, I was actually logging the Firebase Snapshot object instead of the .val().

Node.js - Counting the number of objects in MongoDB collection

I am new to Node.js and MongoDB and I am really struggling to wrap my head around callbacks. I have read a few articles but it is still quite confusing to me. In the code below, I am trying to return the count of orders that have some properties which I have expressed in the query in orderModel.count(query, next):
controllers/order.js:
var mongoose = require ('../config/db');
var orderModel = require('../models/order').model;
var User = require('./user');
var Error = require('../config/error');
createOrder: function (user, order, next) {
if (newOrder.totalPrice > user.credit && orderModel.
count({$and: [{user: order.user}, {active: true}, {$or: [{status: 0}, {status: 1}]}]},
function(err, count){
if(err)
console.log(err);
else
count; }) > 0)
return next({error: Error.InsufficientCredits});
}
I don't think I am correctly obtaining the variable count because when I tried printing out the result of the second condition in the if statement, I got this data printed out:
Query {
_mongooseOptions: {},
mongooseCollection:
NativeCollection {
collection: Collection { s: [Object] },
opts: { bufferCommands: true, capped: false },
name: 'orders',
collectionName: 'orders',
conn:
NativeConnection {
base: [Object],
collections: [Object],
models: [Object],
config: [Object],
replica: false,
hosts: null,
host: 'dsXXXXXX.mlab.com',
port: XXXXXX,
user: 'XXXX',
pass: 'XXXX',
name: 'X',
options: [Object],
otherDbs: [],
_readyState: 1,
_closeCalled: false,
_hasOpened: true,
_listening: false,
db: [Object],
_events: {},
_eventsCount: 0 },
queue: [],
buffer: false,
emitter:
EventEmitter {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined } },
....
You need to put your logic inside the Model.count() method callback function as:
var mongoose = require ('../config/db'),
orderModel = require('../models/order').model,
User = require('./user'),
Error = require('../config/error');
var createOrder = function (user, order, next) {
orderModel.count({
"user": order.user,
"active": true,
"status": { "$in": [0, 1] }
}, function(err, count) { //<-- put logic in this callback
if (err) {
console.log(err);
throw err;
} else if (newOrder.totalPrice > user.credit && count > 0) {
// logic for creating order here
} else {
return next({ "error": Error.InsufficientCredits });
}
})
}

How can I wrap JSON objects in an array using Javascript or Jquery?

How can I wrap all of the data here in an array? This is auto generated from a website system and the app I'm feeding it into requires the data to be in an array. There are multiple sets of data in the JSON, this is just 2 of about 5 or 6.
collection: {
id: "5096f729e4b02d37bef658f2",
enabled: true,
starred: false,
type: 10,
ordering: 3,
title: "About",
navigationTitle: "About",
urlId: "about",
itemCount: 0,
updatedOn: 1347025745523,
publicCommentCount: 0,
folder: false,
dropdown: false,
tags: [ ],
categories: [ ],
homepage: true,
typeName: "page",
synchronizing: false,
typeLabel: "page",
fullUrl: "/"
},
websiteSettings: {
id: "5096f728e4b02d37bef658e0",
websiteId: "5096f728e4b02d37bef658df",
type: "Business",
subject: "Personal",
country: "US",
state: "NY",
markdownMode: false,
simpleLikingEnabled: true,
commerceEnabled: false,
defaultPostFormat: "%y/%m/%d/%t",
commentLikesAllowed: true,
commentAnonAllowed: true,
commentThreaded: true,
commentApprovalRequired: false,
commentAvatarsOn: true,
commentSortType: 2,
commentFlagThreshold: 0,
commentFlagsAllowed: true,
commentEnableByDefault: true,
commentDisableAfterDaysDefault: 0,
disqusShortname: "username",
homepageTitleFormat: "%s - This is a test",
collectionTitleFormat: "%c — %s - This is a test",
itemTitleFormat: "%i — %s - This is a test",
commentsEnabled: true,
allowSquarespacePromotion: false,
storeSettings: {
storeTitle: "Test",
returnPolicy: null,
termsOfService: null,
privacyPolicy: null,
stockLevelAlertLimit: 5,
useLightCart: false,
stripeConnected: false,
storeState: 3
}
}
Okay so assume you have the JSON returned from the API in a variable called rawJson. That is easy enough for you to do with jquery, for example with getJSON. Now you can you acheive what you want with this code:
var rawJson = // get this yourself, pretend I'm inserting the JSON literal from the url you linked to above in your comments
arrayYouWant = [];
for (category in rawJson) {
for (key in rawJson[category]) {
arrayYouWant.push( {key: rawJson[category][key]} )
}
}
You can also flat those two objects and convert them into an array, see the snippet below.
var rawJSON = {
collection: {
id: "5096f729e4b02d37bef658f2",
enabled: true,
starred: false,
type: 10,
ordering: 3,
title: "About",
navigationTitle: "About",
urlId: "about",
itemCount: 0,
updatedOn: 1347025745523,
publicCommentCount: 0,
folder: false,
dropdown: false,
tags: [],
categories: [],
homepage: true,
typeName: "page",
synchronizing: false,
typeLabel: "page",
fullUrl: "/"
},
websiteSettings: {
id: "5096f728e4b02d37bef658e0",
websiteId: "5096f728e4b02d37bef658df",
type: "Business",
subject: "Personal",
country: "US",
state: "NY",
markdownMode: false,
simpleLikingEnabled: true,
commerceEnabled: false,
defaultPostFormat: "%y/%m/%d/%t",
commentLikesAllowed: true,
commentAnonAllowed: true,
commentThreaded: true,
commentApprovalRequired: false,
commentAvatarsOn: true,
commentSortType: 2,
commentFlagThreshold: 0,
commentFlagsAllowed: true,
commentEnableByDefault: true,
commentDisableAfterDaysDefault: 0,
disqusShortname: "username",
homepageTitleFormat: "%s - This is a test",
collectionTitleFormat: "%c — %s - This is a test",
itemTitleFormat: "%i — %s - This is a test",
commentsEnabled: true,
allowSquarespacePromotion: false,
storeSettings: {
storeTitle: "Test",
returnPolicy: null,
termsOfService: null,
privacyPolicy: null,
stockLevelAlertLimit: 5,
useLightCart: false,
stripeConnected: false,
storeState: 3
}
}
}
var myObject = Object.assign({}, rawJSON.collection); // merging objects aka extend
myObject = Object.assign(myObject, rawJSON.websiteSettings);
// {a: 1, b: 2} => [['a', 1], ['b', 2]]
var objAsAnArray=Object.keys(myObject).map((k)=>[k, JSON.stringify(myObject[k])])
console.log(objAsAnArray)

Categories