We Keep Coding

Apify - How to Enqueue URL Variations Efficiently

I am creating a new actor in Apify with Cheerio to read an input file of URLs and return primarily two items: (1) the HTTP status code and (2) the HTML title. As part of our process, I would like to be able to try up to 4 variations of each input URL, such as:
HTTP://WWW.SOMEURL.COM
HTTPS://WWW.SOMEURL.COM
HTTP://SOMEURL.COM
HTTPS://SOMEURL.COM
If one of the 4 variations is successful, then the process should ignore the other variations and move to the next input URL.
I read the original input list into a RequestList, and then would like to create the variations in a RequestQueue. Is this the most efficient way to do it? Please see code below, and thank you!
const Apify = require('apify');
const {
utils: { enqueueLinks },
} = Apify;
const urlParse = require('url');
Apify.main(async () => {
const input = await Apify.getInput();
const inputFile = input.inputFile;
console.log('INPUT FILE: ' + inputFile);
const requestList = await Apify.openRequestList('urls', [
{ requestsFromUrl: inputFile, userData: { isFromUrl: true } },
]);
const requestQueue = await Apify.openRequestQueue();
const proxyConfiguration = await Apify.createProxyConfiguration();
const handlePageFunction = async ({ $, request, response }) => {
let parsedHost = urlParse.parse(request.url).host;
let simplifiedHost = parsedHost.replace('www.', '');
const urlPrefixes = ['HTTP://WWW.', 'HTTPS://WWW.', 'HTTP://', 'HTTPS://'];
let i;
for (i = 0; i < urlPrefixes.length; i++) {
let newUrl = urlPrefixes[i] + simplifiedHost;
console.log('NEW URL: ' + newUrl);
await requestQueue.addRequest({ url: newUrl });
}
console.log(`Processing ${request.url}`);
const results = {
inputUrl: request.url,
httpCode: response.statusCode,
title: $('title').first().text().trim(),
responseUrl: response.url
};
await Apify.pushData(results);
};
const crawler = new Apify.CheerioCrawler({
proxyConfiguration,
maxRequestRetries: 0,
handlePageTimeoutSecs: 60,
requestTimeoutSecs: 60,
requestList,
requestQueue,
handlePageFunction,
handleFailedRequestFunction: async ({ request }) => {
await Apify.pushData({ inputUrl: request.url, httpCode: '000', title: '', responseUrl: ''});
}
});
await crawler.run();
});

you should create your URL list beforehand. the handlePageFunction is only used for the actual scraping part, and you should only have the Apify.pushData there:
//...
const initRequestList = await Apify.openRequestList('urls', [
{ requestsFromUrl: inputFile },
]);
const parsedRequests = [];
let req;
while (req = await initRequestList.fetchNextRequest()) {
const parsedHost = urlParse.parse(req .url).host;
const simplifiedHost = parsedHost.replace('www.', '');
const urlPrefixes = ['HTTP://WWW.', 'HTTPS://WWW.', 'HTTP://', 'HTTPS://'];
for (let i = 0; i < urlPrefixes.length; i++) {
let newUrl = urlPrefixes[i] + simplifiedHost;
console.log('NEW URL: ' + newUrl);
parsedRequests.push({
url: newUrl,
userData: { isFromUrl: true }
});
}
}
const requestList = await Apify.openRequestList('starturls', parsedRequests);
//...
const crawler = new Apify.CheerioCrawler({
proxyConfiguration,
maxRequestRetries: 0,
handlePageTimeoutSecs: 60,
requestTimeoutSecs: 60,
handlePageFunction,
requestList,
handleFailedRequestFunction: async ({ request }) => {
await Apify.pushData({ inputUrl: request.url, httpCode: '000', title: '', responseUrl: ''});
}
});
//...
requestsFromUrl is a greedy function that tries to parse all URLs from to the given resource. so you'll have to perform the processing as an additional step.

How to make api endpoint target user's localhost when deployed to Heroku

I have this api which works fine when running locally. But, once it is deployed to Heroku i get a error 503 which is because it tries to target localhost on Heroku's server and not the user's localhost. Is there a way to make this target the user's localhost instead?
The frontend is React. Here's the code in React that fetches this api every 5sec.
axiosFunc = () => {
const { user } = this.props.auth;
console.log(user);
axios.get(`api/avaya/${user.id}`).then((res) => console.log(res));
};
timer = (time) => {
const date = new Date(time);
return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
};
componentDidMount() {
this.axiosFunc();
this.interval = setInterval(this.axiosFunc, 5000);
}
componentWillUnmount() {
clearInterval(this.interval);
}
and this is the API on the backend with express
const router = require("express").Router();
const xml2js = require("xml2js");
const Avaya = require("../../models/Avaya");
const User = require("../../models/User");
router.route("/:id").get(async (req, res) => {
const user = await User.findById(req.params.id);
const axios = require("axios");
axios({
method: "post",
baseURL: `http://127.0.0.1:60000/onexagent/api/registerclient?name=${user.username}`,
timeout: 2000,
})
.then((reg) => {
xml2js
.parseStringPromise(reg.data, { mergeAttrs: true })
.then((result) => {
if (result.RegisterClientResponse.ResponseCode[0] === "0") {
const clientId = result.RegisterClientResponse.ClientId[0];
user.avayaClientId = clientId;
user.save();
}
const clientId = user.avayaClientId;
axios({
method: "post",
url: `http://127.0.0.1:60000/onexagent/api/nextnotification?clientid=${clientId}`,
}).then((notification) => {
xml2js
.parseStringPromise(notification.data, { mergeAttrs: true })
.then((result) => {
const notifType = [];
const notifDetails = [];
for (let i in result.NextNotificationResponse) {
notifType.push(i);
}
const arranged = {
NotificationType: notifType[1],
ResponseCode:
result.NextNotificationResponse[notifType[0]][0],
};
for (let i in result.NextNotificationResponse[
notifType[1]
][0]) {
notifDetails.push(i);
}
for (let i = 0; i < notifDetails.length; i++) {
arranged[[notifDetails[i]][0]] =
result.NextNotificationResponse[notifType[1]][0][
notifDetails[i]
][0];
}
for (let i in arranged) {
if ("Outbound" in arranged) {
arranged.CallType = "Outbound";
} else if ("Inbound" in arranged)
arranged.CallType = "Inbound";
else {
arranged.CallType = " ";
}
}
if (
arranged.NotificationType === "VoiceInteractionCreated" ||
arranged.NotificationType === "VoiceInteractionMissed" ||
arranged.NotificationType === "VoiceInteractionTerminated"
) {
const newLogs = new Avaya({
notification: arranged,
});
newLogs.owner = user;
newLogs.save();
user.avayaNotifications.push(newLogs),
user
.save()
.then((logs) => res.json(logs))
.catch((err) => res.status(400).json("Error: " + err));
} else {
res.send("Nothing to record");
}
});
});
});
})
.catch((err) => res.status(503).json(err));
});
router.route("/history/:username").get(async (req, res) => {
const user = await User.findOne({ username: [`${req.params.username}`] });
Avaya.find({ owner: [`${await user.id}`] }).then((user) => res.json(user));
});
module.exports = router;
EDIT: I was able to fix thanks to #Molda
using fetch instead of axios doesn't result in cors error.
New frontend code
getLogs = async () => {
const { user } = this.props.auth;
const reg = await fetch(
`http://127.0.0.1:60000/onexagent/api/registerclient?name=${user.id}`
);
let regData = await reg.text();
let regxml = new XMLParser().parseFromString(regData);
if (regxml.attributes.ResponseCode === "0") {
axios.post(`/api/avaya/register/${user.id}`, regxml);
console.log(regxml.attributes.ResponseCode);
}
let resp = await fetch(`/api/avaya/getid/${user.id}`);
let clientId = await resp.text();
let logs = await fetch(
`http://127.0.0.1:60000/onexagent/api/nextnotification?clientid=${clientId}`
);
let data = await logs.text();
var xml = new XMLParser().parseFromString(data);
axios.post(`/api/avaya/getlogs/${user.id}`, xml);
};
timer = (time) => {
const date = new Date(time);
return `${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}`;
};
componentDidMount() {
this.getLogs();
this.interval = setInterval(this.getLogs, 5000);
}
New backend code:
const router = require("express").Router();
const Avaya = require("../../models/Avaya");
const User = require("../../models/User");
router.route("/register/:id").post(async (req, res) => {
const user = await User.findById(req.params.id);
const clientId = req.body.attributes.ClientId;
user.avayaClientId = clientId;
user.save();
});
router.route("/getid/:id").get(async (req, res) => {
const user = await User.findById(req.params.id);
res.send(user.avayaClientId);
});
router.route("/getlogs/:id").post(async (req, res) => {
const user = await User.findById(req.params.id);
const arranged = {
NotificationType: req.body.children[0].name,
ResponseCode: req.body.attributes.ResponseCode,
CallType: " ",
};
for (let i in req.body.children[0].attributes) {
if (i === "Outbound") {
arranged.CallType = "Outbound";
}
if (i === "Inbound") {
arranged.CallType = "Inbound";
}
arranged[i] = req.body.children[0].attributes[i];
}
console.log(arranged);
if (
arranged.NotificationType === "VoiceInteractionCreated" ||
arranged.NotificationType === "VoiceInteractionMissed" ||
arranged.NotificationType === "VoiceInteractionTerminated"
) {
const newLogs = new Avaya({
notification: arranged,
});
newLogs.owner = user;
newLogs.save();
user.avayaNotifications.push(newLogs),
user
.save()
.then((logs) => res.json(logs))
.catch((err) => res.status(400).json("Error: " + err));
} else {
res.send("Nothing to record");
}
});
router.route("/history/:username").get(async (req, res) => {
const user = await User.findOne({ username: [`${req.params.username}`] });
Avaya.find({ owner: [`${await user.id}`] }).then((user) => res.json(user));
});
module.exports = router;

I really don't get the part of (requesting with Axios in API)
Is this a third party API ?
But I suggest you to use (.env) which is a file in your root folder contains the development config like base URLs, expire tokens, API keys ... etc
and when you upload to Heroku you have to make a (.env) in Heroku app and but your config
Let's take an example
in my development mode, my .env looks like
app_url = localhost:4000
port = 4000
db = development_api
db_username = root
db_password =
db_engine = mysql2
in my production mode, my .env looks like
app_url = http://appsomething.heroku.com
port = 80
db = production_api
db_username = root
db_password = 3210LDWAK#AALKQ
db_engine = mysql2
and read more about how to use .ENV

error firebase functions [Promises must be handled appropriately] on deploy

I was written a code previous week and it deploys without any error on firebase server. but now I cannot deploy it again on another account in orders to I don't change my code!
one of my friends tell me this in about new update of firebase but I don't find any solution for this!
it shows these errors
Promises must be handled appropriately
and
block is empty
the first error pointed to my first line and the second one pointed to end 'catch' block :
import * as functions from 'firebase-functions';
import * as admin from 'firebase-admin';
admin.initializeApp();
// export const helloWorld = functions.https.onRequest((request, response) => {
// console.log("sadegh");
// response.send("Hello from Firebase1!");
// });
//
export const sendChatNotification = functions
.firestore.document('rooms/{roomId}/messages/{messageId}')
.onCreate((snap, context) => {
const roomId = context.params.roomId;
const messageId = context.params.messageId;
const newValue = snap.data();
const receiverId = newValue.receiverId;
const text = newValue.text;
const type = newValue.type;
const senderName = newValue.senderName;
var p = admin.firestore().collection("users").doc(receiverId).get();
p.then(snapshot2 => {
const data2 = snapshot2.data();
const firebaseNotificationToken = data2.firebaseNotificationToken;
// const name = data2.name;
if (type == 'voiceCall' || type == 'videoCall' || type == 'hangUp') {
const channelId = newValue.channelId;
const senderId = newValue.senderId;
const status = newValue.status;
console.log("type: " + type + " /status: " + status)
let message = {
data: {
type: type,
senderId: senderId,
senderName: senderName,
receiverId: receiverId,
status: status,
channelId: channelId,
roomId: roomId
},
token: firebaseNotificationToken
};
sendMessage(message)
if (status == "canceled") {
let message1 = {
notification: {
title: '☎ Missed voice call ',
body: senderName
},
token: firebaseNotificationToken
};
sendMessage(message1)
} else if ((type == 'voiceCall' || type == 'videoCall') && status = '') {
let message1 = {
notification: {
title: '☎ ' + senderName + ' is calling you',
body: 'tap to answer...'
},
token: firebaseNotificationToken
};
sendMessage(message1)
}
} else {
let message = {
notification: {
title: '📃 ' + senderName,
body: text
},
token: firebaseNotificationToken
};
sendMessage(message)
}
return "";
}).catch((e) => {
console.log('error: ' + e);
return null;
});
// return "";
// }).catch(e=>{console.log('error: '+e)});
return "sadegh";
});
function sendMessage(message) {
admin.messaging().send(message)
.then((response) => {
// Response is a message ID string.
console.log('Successfully sent message:', response);
})
.catch((error) => {
console.log('Error sending message:', error);
});
}

Your code is a bit messy and it is not really easy to understand it without dedicating a long time.
However, here is below a piece of code that should work and that cover one case of your Business Logic. Note how the promises returned by the asynchronous tasks are returned.
export const sendChatNotification = functions.firestore
.document('rooms/{roomId}/messages/{messageId}')
.onCreate((snap, context) => {
const roomId = context.params.roomId;
const messageId = context.params.messageId;
const newValue = snap.data();
const receiverId = newValue.receiverId;
const text = newValue.text;
const type = newValue.type;
const senderName = newValue.senderName;
var p = admin
.firestore()
.collection('users')
.doc(receiverId)
.get();
return p.then(snapshot2 => { // <- HERE, the promise is returned
const data2 = snapshot2.data();
const firebaseNotificationToken = data2.firebaseNotificationToken;
if (type == 'voiceCall' || type == 'videoCall' || type == 'hangUp') {
const channelId = newValue.channelId;
const senderId = newValue.senderId;
const status = newValue.status;
console.log('type: ' + type + ' /status: ' + status);
let message = {
data: {
type: type,
senderId: senderId,
senderName: senderName,
receiverId: receiverId,
status: status,
channelId: channelId,
roomId: roomId
},
token: firebaseNotificationToken
};
return admin.messaging().send(message); // <- HERE, the promise is returned
}
});
});
I would suggest you watch the 3 videos about "JavaScript Promises" from the Firebase video series: https://firebase.google.com/docs/functions/video-series/

The problem is you commented the return in your catch block
As your Firebase .get() function must return a promise, in your code, if it fails, it won't return a promise and it will hang there.
either use return null or return something to be handled by the calling app

Can't figure out why my app.get is being run twice?

I have a app.get which inside of it is quite a bit of logic. Which everything works great aside from some of the logic being called twice for some reason. I have noticed when I was saving something to by db that it would save two rows.
So I put a console.log in that area and sure enough it was logging it twice.
Any reason why this is happening?
app.get('/shopify/callback', (req, res) => {
const { shop, hmac, code, state } = req.query;
const stateCookie = cookie.parse(req.headers.cookie).state;
if (state !== stateCookie) {
return res.status(403).send('Request origin cannot be verified');
}
if (shop && hmac && code) {
// DONE: Validate request is from Shopify
const map = Object.assign({}, req.query);
delete map['signature'];
delete map['hmac'];
const message = querystring.stringify(map);
const providedHmac = Buffer.from(hmac, 'utf-8');
const generatedHash = Buffer.from(
crypto
.createHmac('sha256', config.oauth.client_secret)
.update(message)
.digest('hex'),
'utf-8'
);
let hashEquals = false;
try {
hashEquals = crypto.timingSafeEqual(generatedHash, providedHmac)
} catch (e) {
hashEquals = false;
};
if (!hashEquals) {
return res.status(400).send('HMAC validation failed');
}
// DONE: Exchange temporary code for a permanent access token
const accessTokenRequestUrl = 'https://' + shop + '/admin/oauth/access_token';
const accessTokenPayload = {
client_id: config.oauth.api_key,
client_secret: config.oauth.client_secret,
code,
};
request.post(accessTokenRequestUrl, { json: accessTokenPayload })
.then((accessTokenResponse) => {
const accessToken = accessTokenResponse.access_token;
// DONE: Use access token to make API call to 'shop' endpoint
const shopRequestUrl = 'https://' + shop + '/admin/shop.json';
const shopRequestHeaders = {
'X-Shopify-Access-Token': accessToken,
}
request.get(shopRequestUrl, { headers: shopRequestHeaders })
.then((shopResponse) => {
const response = JSON.parse(shopResponse);
const shopData = response.shop;
console.log('BEING CALLED TWICE...')
res.render('pages/brand_signup',{
shop: shopData.name
})
})
.catch((error) => {
res.status(error.statusCode).send(error.error.error_description);
});
})
.catch((error) => {
res.status(error.statusCode).send(error.error.error_description);
});
} else {
res.status(400).send('Required parameters missing');
}
});

How do you download an attachment with Gmail API with nodejs?

Im trying to make a node app on my local computer that does this:
1) Looks at my gmail inbox.
2) If an email has a certain label and an attachment, download it.
Im on step 2.
The part Im confused about is the part about 'parts'.
https://developers.google.com/gmail/api/v1/reference/users/messages/attachments/get
Googles sample code:
function getAttachments(userId, message, callback) {
var parts = message.payload.parts;
for (var i = 0; i < parts.length; i++) {
var part = parts[i];
if (part.filename && part.filename.length > 0) {
var attachId = part.body.attachmentId;
var request = gapi.client.gmail.users.messages.attachments.get({
'id': attachId,
'messageId': message.id,
'userId': userId
});
request.execute(function(attachment) { // <--- that attachment param
callback(part.filename, part.mimeType, attachment);
});
}
}
}
It seems that 'attachment' contains the attachment data.
Here is my code:
const gmail = google.gmail({version: 'v1', auth});
gmail.users.messages.list({
userId: 'me',
'q':'subject:my-test has:nouserlabels has:attachment'
}, (err, res) => {
if (err) return console.log('1 The API returned an error: ' + err);
const msgs = res.data.messages;
if(typeof msgs !== 'undefined'){
console.log('Messages:');
msgs.forEach( msg => {
console.log(`- ${msg.id}`);
gmail.users.messages.get({
userId: 'me',
id: msg.id,
format:'full'
},(err, res) => {
if (err) return console.log('2 The API returned an error: ' + err);
// Wheres the data????? //
console.log('A')
console.log(res.data.payload.parts)
console.log('B')
console.log(res.data.payload.parts[1])
console.log('C')
console.log(res.data.payload.parts[1].body)
})
});
} else {
console.log('no messages found')
}
});

I've just used this to retrieve the docs from my favorite accountant.
const fsPromises = require('fs/promises');
const path = require('path');
const {google} = require('googleapis');
const readline = require('readline');
const fs = require('fs');
.... code from https://developers.google.com/gmail/api/quickstart/nodejs for auth
const fromEmail = process.argv[2];
const shortFromEmail = fromEmail.substring(0, fromEmail.indexOf('#'));
/**
* Lists the labels in the user's account.
*
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
async function main(auth) {
const gmail = google.gmail({version: 'v1', auth})
// console.log('list messages with attachments auth: ' + JSON.stringify(auth))
const {data} = await gmail.users.messages.list({
userId: 'me',
'q':`from:${fromEmail} has:nouserlabels has:attachment`,
// maxResults: 5
});
console.log(data);
let {messages} = data;
await messages.reduce(async function (accProm, messageId) {
await accProm;
const {data: message} = await gmail.users.messages.get({
userId: 'me',
id: messageId.id,
format: 'full'
});
// console.log(JSON.stringify(message, null, 2));
let messagePayloadParts = message.payload.parts
const attachments = await messagePayloadParts.reduce(async function (acc2Prom, part) {
const acc2 = await acc2Prom;
if (!part.body.size || part.body.attachmentId == undefined) {
return acc2;
}
const internalDate = new Date(parseInt(message.internalDate, 10));
const datestring = internalDate.getFullYear() + " " + (internalDate.getMonth()+1).toString().padStart(2, "0") + " " + internalDate.getDate().toString().padStart(2, "0");
console.log(datestring, shortFromEmail, part.filename, part)
let fileExt;
switch (part.mimeType) {
case 'application/octet-stream': fileExt = ''; break;
case 'application/pdf': fileExt = 'pdf'; break;
case 'message/rfc822': fileExt = 'eml'; break;
case 'image/jpeg': fileExt = 'jpg'; break;
case 'image/png': fileExt = 'png'; break;
case 'application/msword': fileExt = 'doc'; break;
default: console.error('unknownMimeType', part.mimeType); boom;
}
const dirname = `${datestring} ${shortFromEmail}`;
const filename = part.filename ? part.filename : `noname_${part.body.attachmentId.substring(0,8)}.${fileExt}`
const attachmentPath = path.join(dirname, filename);
const s = await fsPromises.stat(attachmentPath).catch(e => undefined);
if (s) { return acc2 };
const {data: attachment} = await gmail.users.messages.attachments.get({
userId: 'me',
messageId: message.id,
id: part.body.attachmentId,
});
const { size, data: dataB64 } = attachment;
await fsPromises.mkdir(dirname, {recursive: true});
await fsPromises.writeFile(attachmentPath, Buffer.from(dataB64, 'base64'));
return acc2;
// console.log('callback: ' + JSON.stringify(attachments))
}, Promise.resolve([]));
}, Promise.resolve());
}