How to scrape Music Charts & Insights page from charts.youtube.com? - javascript

I used got and request-promise modules for scraping, but I got a spinner circle in response.
Basically, I want get video items not spinner circle.
/* This example use request-promise */
const rp = require('request-promise');
const url = 'https://charts.youtube.com/charts/TrendingVideos/gb';
rp(url)
.then(function (html) {
//success!
fs.writeFileSync('./index.html', html);
})
.catch(function (err) {
//handle error
});
/* This example use got */
got('https://charts.youtube.com/').then(response => {
fs.writeFileSync('./index.html', response.body);
const dom = new JSDOM(response.body);
dom.window.document.querySelectorAll('a').forEach(link => {
console.log(link.href);
});
}).catch(err => {
console.log(err);
});
Can someone help me?

The data comes from an API on :
POST https://charts.youtube.com/youtubei/v1/browse?alt=json&key=${apiKey}
You need to get the api key from the html using regex and then perform the call with the required input data :
const got = require("got");
const util = require("util");
(async () => {
let response = await got(
"https://charts.youtube.com/charts/TrendingVideos/gb"
);
let keyRegex = /\"INNERTUBE_API_KEY\"\s*:\s*\"(.*?)\"/gm;
let apiKey = keyRegex.exec(response.body)[1];
response = await got.post(`https://charts.youtube.com/youtubei/v1/browse?alt=json&key=${apiKey}`,
{
json: {
context: {
client: {
clientName: "WEB_MUSIC_ANALYTICS",
clientVersion: "0.2",
hl: "en",
gl: "en",
experimentIds: [],
experimentsToken: "",
theme: "MUSIC",
},
capabilities: {},
request: {
internalExperimentFlags: [],
},
},
browseId: "FEmusic_analytics_charts_home",
query:
"chart_params_type=WEEK&perspective=CHART&flags=viral_video_chart&selected_chart=TRACKS&chart_params_id=weekly%3A0%3A0%3Agb",
},
responseType: "json",
headers: {
referer: "https://charts.youtube.com/charts/TrendingVideos/gb",
},
}
);
videoTrend = response.body.contents.sectionListRenderer.contents[0].musicAnalyticsSectionRenderer.content.videos;
console.log(util.inspect(videoTrend, { showHidden: false, depth: null }));
})();
Try this on repl.it

Related

Amplify lambda function "Not Authorized to access [function] on type [model]"

I am following the explanations from this website: https://aws.amazon.com/de/getting-started/hands-on/build-flutter-mobile-app-part-two/module-four/
If I start the application, a profile is not created, which should be happening in "custom.js".
const { Sha256 } = require("#aws-crypto/sha256-js");
const { defaultProvider } = require("#aws-sdk/credential-provider-node");
const { SignatureV4 } = require("#aws-sdk/signature-v4");
const { HttpRequest } = require("#aws-sdk/protocol-http");
const { default: fetch, Request } = require("node-fetch");
const GRAPHQL_ENDPOINT = process.env.API_AMPLIFYTRIPSPLANNER_GRAPHQLAPIENDPOINTOUTPUT;
const AWS_REGION = process.env.AWS_REGION || 'us-east-1';
const query = /* GraphQL */ `
mutation createProfile($email: String!,$owner: String!) {
createProfile(input: {
email: $email,
owner: $owner,
}) {
email
}
}
`;
/**
* #type {import('#types/aws-lambda').PostConfirmationTriggerHandler}
*/
exports.handler = async (event) => {
console.log(`EVENT: ${JSON.stringify(event)}`);
const variables = {
email: event.request.userAttributes.email,
owner: `${event.request.userAttributes.sub}::${event.userName}`
};
const endpoint = new URL(GRAPHQL_ENDPOINT);
const signer = new SignatureV4({
credentials: defaultProvider(),
region: AWS_REGION,
service: 'appsync',
sha256: Sha256
});
const requestToBeSigned = new HttpRequest({
method: 'POST',
headers: {
'Content-Type': 'application/json',
host: endpoint.host
},
hostname: endpoint.host,
body: JSON.stringify({ query, variables }),
path: endpoint.pathname
});
const signed = await signer.sign(requestToBeSigned);
const request = new Request(endpoint, signed);
let statusCode = 200;
let body;
let response;
try {
response = await fetch(request);
body = await response.json();
if (body.errors) statusCode = 400;
} catch (error) {
statusCode = 500;
body = {
errors: [
{
message: error.message
}
]
};
}
console.log(`statusCode: ${statusCode}`);
console.log(`body: ${JSON.stringify(body)}`);
return {
statusCode,
body: JSON.stringify(body)
};
};
I was expecting the profile to be created, when I run the application. But there is no profile created.
Cloud watch is providing this error log message:
{
"data": {
"createProfile": null
},
"errors": [
{
"path": [
"createProfile"
],
"data": null,
"errorType": "Unauthorized",
"errorInfo": null,
"locations": [
{
"line": 3,
"column": 6,
"sourceName": null
}
],
"message": "Not Authorized to access createProfile on type Profile"
}
]
}
I am trying to understand the two problems:
The data should not be null
There should not be an authorization problem (which maybe a follow-on from the data issue)
I have not found any useful hints yet.
What needs to be changed?
I now realized my error. I did not follow the instructions to start the second tuturial within a clean directory. The Amplify configuration required a change in the Auth module, that can only be implemented in a new project (you cannot change auth type after deploying an app).

TypeError: axios.post(...).then is not a function

I have following Node.js script.
const axios = require('axios');
const https = require('https');
const postRequest = (url, data) => {
gLogger.debug('postRequest started');
// try {
const headers = {
'Content-Type': 'application/json',
'Content-Length': JSON.stringify(data).length,
};
const load = {
headers,
httpsAgent: agent,
};
gLogger.debug(`postRequest load: ${JSON.stringify(load)}`);
const result = axios.post(url, data, load).then((response) => {
return result;
})
.catch((error) => {
return error;
});
};
And this is for unit test:
const axios = require('axios');
const personalRecords = {
data: { peopleDomain: { paom: { data: { persons: [], delta: 1, recordsFetched: 10 } } } },
};
const tockenData = {
data: {
access_token: 'access_token',
expires_in: 1000,
},
};
// jest.useFakeTimers();
jest.setTimeout(8000);
jest.mock('axios', () => ({
post: jest.fn().mockReturnValue(tockenData),
get: jest.fn().mockReturnValue(personalRecords),
defaults: jest.fn().mockReturnValue(),
}));
The problem when I am running unit test yarn test, I keep getting the following error:
TypeError: axios.post(...).then is not a function.
What is the problem and how to fix it?
This is because you mock post function to be a function that returns a value instead of a promise. Remember post returns promise
This is the line that causes trouble:
post: jest.fn().mockReturnValue(tockenData),
To mock axios, there is an answer here:
How do I test axios in Jest?

why patch function not calling react native to node js

React Native
this my code. when i request for data push in database then react native App.js fetch function is not execute you can see my code and solve this bug...
this is my react native screen code where i function is not calling backend
App.js
const postdata = async () => {
const data = {cname, cphone, ccity, caddress, cemail, gender}
// 63064232cf92b07e37090e0a
const res = await fetch(`http://192.168.43.220:8000/Newcustomer/${id}`, {
method: "PATCH",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
cname, cphone, ccity, caddress, cemail, gender
})
})
const data2 = await res.json();
console.log(data2);
if (!data2) {
window.alert('error in get data2');
}
else {
setdata(data2);
window.alert("Costomer Added")
navigation.navigate("Home")
}
}
Back End
Node js
this is my node js code. it code is properly working when i try to postman
app.js
app.patch('/Newcustomer/:id', async (req, res) => {
const _id = req.params.id;
getschema.findByIdAndUpdate(_id, {
$push: {
costomer:
{
cname: req.body.cname,
cphone: req.body.cphone,
ccity: req.body.ccity,
caddress: req.body.caddress,
cemail: req.body.cemail,
gender: req.body.gender
},
}
})
.then(data => {
res.status(201).json(data);
}).catch(err => {
console.log(err);
})
})

Node js Async/Await and return not working as expected

I'm trying to build a simple report server with node express. But, it is not functioning as expected.
This is my api end point to generate report:
router.post('/test', async function (req, res) {
return res.json(await reportService.test());
});
This is service layer:
var pdf = require("pdf-creator-node");
var fs = require("fs");
var path = require("path");
var base64Util = require("../utils/base64util");
async function test() {
var html = fs.readFileSync(path.join(__dirname, "../templates/test.html"), "utf8");
var options = {
format: "A3",
orientation: "portrait",
border: "10mm",
header: {
height: "45mm",
contents: '<div style="text-align: center;">header</div>'
},
footer: {
height: "28mm",
contents: {
first: 'Cover page',
2: 'Second page',
default: '<span style="color: #444;">{{page}}</span>/<span>{{pages}}</span>',
last: 'Last Page'
}
}
};
var users = [{
name: "Shyam",
age: "26",
},
{
name: "Navjot",
age: "26",
},
{
name: "Vitthal",
age: "26",
},
];
var document = {
html: html,
data: {
users: users
},
path: path.join(__dirname, "../reports/test.pdf"),
type: "",
};
pdf
.create(document, options)
.then(async (res) => {
logger.info("Report Generated: " + res.filename);
let base64 = await base64Util.convert("../reports/test.pdf");
return {
success: true,
url: res.filename,
base64: base64
};
})
.catch((error) => {
logger.error(error);
return {
success: false
};
});
}
module.exports = {
test
}
This is pdf to base64 converting method:
const pdf2base64 = require('pdf-to-base64');
var path = require("path");
async function convert(filePath){
pdf2base64(path.join(__dirname, filePath))
.then(
(response) => {
return response;
}
)
.catch(
(error) => {
logger.log(error);
return false;
}
)}
module.exports = {
convert
}
I want send the response after generating the pdf and then converting it to base64. But it didn't return anything.
Your test method doesn't wait for anything, because you haven't told it to. Neither it returns anything. Add return in front of your pdf promise:
return pdf
.create(document, options)
...
The same applies to pdf2base64 call.
function convert(filePath) {
return pdf2base64(path.join(__dirname, filePath))
...
}
Also, you specify those methods to be async, but you aren't actually making any use of async/await promises. You can get rid of them, traditional Promises don't need them.
https://developer.mozilla.org/en-US/docs/Learn/JavaScript/Asynchronous/Async_await
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
Your test function needs to return something. Add a return before pdf.create(document...)

In azure functions (js) POST request after async/await call does not work

I am currently working with azure functions in javascript. In my function, I am first getting a specific element from my CosmoDB (this is the async/await part). I get a result and then I want to do an https POST request. However, my problem is, that it never finished the HTTPs request and I don't really know why. What am I doing wrong?
(As you can see I tried 2 different ways of doing the request, once with the standard https function and the commented out the part with npm request package. However, both ways won't work).
Here is my code:
const CosmosClient = require('#azure/cosmos').CosmosClient;
var https = require('https');
var http = require('http');
var request = require('request');
const endpoint = "someEndpoint";
const masterKey = "anymasterkey";
const database = {
"id": "Database"
};
const container = {
"id": "Container1"
};
const databaseId = database.id;
const containerId = container.id;
const client = new CosmosClient({
endpoint: endpoint,
auth: {
masterKey: masterKey
}
});
module.exports = function (context, req) {
const country = "de";
const bban = 12345678;
const querySpec = {
query: "SELECT * FROM Container1 f WHERE f.country = #country AND f.bban = #bban",
parameters: [{
name: "#country",
value: country
},
{
name: "#bban",
value: bban
}
]
};
getContainers(querySpec).then((results) => {
const result = results[0];
context.log('here before request');
var options = {
host: 'example.com',
port: '80',
path: '/test',
method: 'POST'
};
// Set up the request
var req = http.request(options, (res) => {
var body = "";
context.log('request');
res.on("data", (chunk) => {
body += chunk;
});
res.on("end", () => {
context.res = body;
context.done();
});
}).on("error", (error) => {
context.log('error');
context.res = {
status: 500,
body: error
};
context.done();
});
req.end();
// request({
// baseUrl: 'someURL',
// port: 443,
// uri: 'someuri',
// method: 'POST',
// headers: {
// 'Content-Type': 'text/xml;charset=UTF-8',
// 'SOAPAction': 'someaction'
// },
// function (error, response, body) {
// context.log('inside request')
// if (error) {
// context.log('error', error);
// } else {
// context.log('response');
// }
// }
// })
})
};
async function getContainers(querySpec) {
const {container, database} = await init();
return new Promise(async (resolve, reject) => {
const {
result: results
} = await container.items.query(querySpec).toArray();
resolve(results);
})
}
async function init() {
const {
database
} = await client.databases.createIfNotExists({
id: databaseId
});
const {
container
} = await database.containers.createIfNotExists({
id: containerId
});
return {
database,
container
};
}
The last thing that happens is the print of "here before request". After that the function just does nothing until it timesout. So what am I doing wrong? Can't I just this combination of await/async and requests?
As commented you are not sending any data to the POST call. You need to have a req.write before the req.end
req.write(data);
req.end();
That is why the POST call is failing for you. After this fix, it should work

Categories