I am trying to get a url from the get request params.
How to Construct request URL from environment variables
In Pre-request script urlwfsservice and orderid is set from environment variables
{{urlwfsservice}}/v1/merchantorders/{{orderId}}/BoardingActivities.updateMerchantToHub.REQUEST/details/
When use
var urlnew =request.url;
console.log(request.url);
I get this output as variables name not actual value or url
{{urlwfsservice}}/v1/merchantorders/{{orderId}}/BoardingActivities.updateMerchantToHub.REQUEST/details/
how can I get output like below simpleurl ?
var simpleurl = “https://dev-someweburl.com/v1/merchantorders/ZN2aB/BoardingActivities.updateMerchantToHub.REQUEST/details/”;
Complete Pre-request script Code
// how to Construct request URL from environment variables
console.log("logging url");
var urlnew =request.url;
console.log(urlnew);
//var url = "https://dev-someweburl.com/v1/merchantorders/ZN2aB/BoardingActivities.updateMerchantToHub.REQUEST/details/";
var retryDelay = 200;
var retryLimit = 5;
function isProcessingComplete(retryCount) {
pm.sendRequest(urlnew, function (err, response) {
if(err) {
// hmmm. Should I keep trying or fail this run? Just log it for now.
console.log(err);
} else {
// I could also check for response.json().results.length > 0, but that
// would omit SUCCESS with empty results which may be valid
if(response.json().auditRecords.length === 0) {
if (retryCount < retryLimit) {
console.log('Job is still PENDING. Retrying in ' + retryDelay + 'ms');
setTimeout(function() {
isProcessingComplete(++retryCount);
}, retryDelay);
} else {
console.log('Retry limit reached, giving up.');
postman.setNextRequest(null);
}
}
}
});
}
isProcessingComplete(1);
console.log(pm.variables.replaceIn(pm.request.url.toString()))
you can use replaceIn method to replace variables to its actual values. Also use pm. as pm is the new api or syntax in postman
Related
We are using the Apify Web Scraper actor to create a URL validation task that returns the input URL, the page's title, and the HTTP response status code. We have a set of 5 test URLs we are using: 4 valid, and 1 non-existent. The successful results are always included in the dataset, but never the failed URL.
Logging indicates that the pageFunction is not even reached for the failed URL:
2021-05-05T14:50:08.489Z ERROR PuppeteerCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue {"url":"http://www.invalidurl.com","retryCount":1,"id":"XS9JTk8dYRM8bpM"}
2021-05-05T14:50:08.490Z Error: gotoFunction timed out after 30 seconds.
2021-05-05T14:50:08.490Z at PuppeteerCrawler._handleRequestTimeout (/home/myuser/node_modules/apify/build/crawlers/puppeteer_crawler.js:387:15)
2021-05-05T14:50:08.496Z at PuppeteerCrawler._handleRequestFunction (/home/myuser/node_modules/apify/build/crawlers/puppeteer_crawler.js:329:26)
Eventually it times out, based on our settings:
2021-05-05T14:50:42.052Z ERROR Request http://www.invalidurl.com failed and will not be retried anymore. Marking as failed.
2021-05-05T14:50:42.052Z Last Error Message: Error: gotoFunction timed out after 30 seconds.
I tried wrapping the code in the pageFunction in a try/catch block, but again, because the pageFunction is not getting reached for the invalid URL, that does not resolve the issue. Is there a way to still include the failed result in the dataset with a hard-coded response status code of '000'? (See pageFunction code below.) Please let me know if I can provide any additional information, and thanks in advance!
async function pageFunction(context) {
context.log.info("Starting pageFunction");
// use jQuery as $
const { request, jQuery: $ } = context;
const { url } = request;
context.log.info("Trying " + url);
let title = null;
let responseCode = null;
try {
context.log.info("In try block for " + url);
title = $('title').first().text().trim();
responseCode = context.response.status;
} catch (error) {
context.log.info("EXCEPTION for " + url);
title = "";
responseCode = "000";
}
return {
url,
title,
responseCode
};
}
you can use https://sdk.apify.com/docs/typedefs/puppeteer-crawler-options#handlefailedrequestfunction:
you can then push it to the when all retries fail:
handleFailedRequestFunction: async ({ request }) => {
// failed all retries
await Apify.pushData({ url: request.url, responseCode: '000' });
}
I use Javascript to retrieve through a JSON api call the amount of active products pasted in a certain filter.
My typicall response would be
{"total":34,"product_ids":["PRODUCT1","PRODUCT2",....."]}
My script is working fine when products are present but when none of the products are active the response will be:
{"error":"No products found, please check request settings"}
In this case the script will crash.
What I tried to do is to set the var NumEdPicks to 0 when I get an error but I don't really know how as the script is crashing when it doesn't find "total".
This is what the retrieve part of the script looks like
// Retrieve
var url = 'http://api.jetlore.com/products/products_by_filter.json?jl_cid=' + clientID + '&filter=' + filterName + '&per_page=' + maxCount + '&page=1';
var response = HTTP.Get(url);
var responseObj = Platform.Function.ParseJSON(response["Content"]);
var NumEditorsPick = responseObj.total;
if(NumEditorsPick>maxCount){ var NumEditorsPick = maxCount;}
I would like to set NumEditorsPick to 0 when I get the error response.
Some things I was thinking about but which isn't working:
var NumEditorsPick = responseObj.total || 0
or
var NumEditorsPick = ‘total’ in responseObj ? responseObj.total : 0
How to define NumEditorsPick when there is no total?
I've tried so far:
if (responseObj.hasOwnProperty('total')){
var NumEditorsPick = responseObj.total;
}else{
var NumEditorsPick = 0;
}
And
if (responseObj.has("total")){var NumEditorsPick = responseObj.total;
}
if (responseObj.has("error")){var NumEditorsPick = 0;
}
Both are crashing the execution of my script, so I'm starting to think that when there is an error response it just stops the script and ignores the rest, would that be possible? In that case, how to ignore this error response?
EDIT:
After using the try/catch method as suggested in the comments, I managed to finally make it work:
var NumEditorsPick;
try {
var response = HTTP.Get(url);
var responseObj = Platform.Function.ParseJSON(response["Content"]);
NumEditorsPick = responseObj.total;
} catch (error) {
NumEditorsPick = 0;
}
You can use Javascript's hasOwnProperty() to check if the parse JSON has the key you're looking for.
In this case, I'd be something like:
var responseObj = Platform.Function.ParseJSON(response["Content"]);
if (responseObj.hasOwnProperty('error')){
// handle error msg
}else{
// do something else
}
Here's a simple example using the JSON input you've provided.
Update
Ok, so my initial answer was based on what you said here:
My script is working fine when products are present but when none of
the products are active the response will be:
{"error":"No products found, please check request settings"}
But the service you're calling does not return a JSON string containing the error. Instead it returns a 404 and therefore, any attempt to parse or use the response content is not valid.
So, to start, you could try wrapping your HTTP.Get(url)in a try/catch method and on the catch clause set the NumEdPicks to zero.
Another option would be to check HTTP.Get() method documentation to see if the response object has a status (e.g: response.Status) or if you can pass a callback function for response and error, like this example in AJAX:
$.ajax({
url: 'yourUrl',
type: 'GET',
success: function(data){
// Set NumEdPicks to total and do other stuff
},
error: function(data) {
// Set NumEdPicks to zero
}
});
Hi I am trying to retrieve some data from webservice using AngularJS $http get.
I have the following code snippet:
In the servicesjs:
.factory('BoothDesignatedCoordsService', ['$http', function ($http) {
var factory = {};
factory.getBoothDesignatedCoords = function (strBoothName, intFloorPlanID) {
var sendToWS;
var boothDesignatedCoords
var JSONObj = {
BoothName: strBoothName,
FloorPlanID: intFloorPlanID
};
sendToWS = JSON.stringify(JSONObj)
var urlBase = 'http://localhost:4951/wsIPS.asmx/fnGetBoothDesignatedCoords?objJSONRequest=' + sendToWS;
return $http.get(urlBase)
}
return factory;
}])
In the controllerjs:
var boothDesignatedCoords = BoothDesignatedCoordsService.getBoothDesignatedCoords(strListShortListedBooth[i], 3).success(function (response, data, status) {
console.log("successfully send booth name and floor plan id to ws");
console.log("data " + data + " , status : " + status);
console.log("data " + data);
boothDesignatedCoords = data;
for (var c = 0; c < boothDesignatedCoords.length; c += 2) {
}
The $http get is successful as I am able to print "successfully send booth name and floor plan id to ws" in the browser console log.
When I tried to print console.log("data " + data), it gives me some values of an integer array. That is exactly what I want. But in the controller I tried to assign data to the variable boothDesignatedCoords, the program does not run the for loop. Am I missing some code?
EDIT:
I tried to trace the code ( trace the variable called "data" in the controllerjs) and it says "data is not defined"
You appear to be confused about the methods available on the $http promise and their arguments. Try this
BoothDesignatedCoordsService.getBoothDesignatedCoords(strListShortListedBooth[i], 3)
.then(function(response) {
var data = response.data
var status = response.status
console.log('data', data) // note, no string concatenation
// and so on...
})
FYI, the success and error methods have been deprecated for some time and removed from v1.6.0 onwards. Don't use them.
I also highly recommend passing query parameters via the params config object
var urlBase = 'http://localhost:4951/wsIPS.asmx/fnGetBoothDesignatedCoords'
return $http.get(urlBase, {
params: { objJSONRequest: sendToWS }
})
This will ensure the key and value are correctly encoded.
So, I'm defining a cloud function that's supposed to make a call to the foursquare api and generate a list of restaurants (each restaurant is a ParseObject) from the returned JSON. I successfully do this, but I run into problems when trying to save these objects to my database and send them back to my phone by calling response.success(). The large code block below saves the list to my database, but if I try
Parse.Object.saveAll(restaurants)
response.success(restaurants)
I end the function before all of the restaurants are saved. I tried using this line instead
Parse.Object.saveAll(restaurants).then(response.success(restaurants))
, but only half of the restaurants get saved before I get the error "Failed with: Uncaught Tried to save an object with a pointer to a new, unsaved object." I also get this error if I call response.success(restaurants) without attempting to save the list. I read that this is a bug in parse preventing someone from printing or passing unsaved ParseObjects. Any ideas? I also tried using .then on the http request, but I get the same issues or a new error: "com.parse.ParseException: i/o failure: java.net.SocketTimeoutException: Read timed out. "
Parse.Cloud.define("callFourSquare", function(request, response) {
//The Parse GeoPoint with current location for search
var geo = request.params.location;
var geoJson = geo.toJSON();
var url = "https://api.foursquare.com/v2/venues/explore?ll=" + geoJson.latitude + ","
+ geoJson.longitude + "§ion=food&sortByDistance=1&limit=50&venuePhotos=1&categoryId=4d4b7105d754a06374d81259&client_id= C043AJBWKIPBAXOHLPA0T40SG5L0GGMQRWQCCIKTRRVLFPTH"
+ "&client_secret=Y1GZZRHXEW1I3SQL3LTHQFNIZRDCTRG12FVIQI5QGUX0VIZP&v=20140715";
console.log(url);
//Call to FourSquare api, which returns list of restaurants and their details
Parse.Cloud.httpRequest({
method: "GET",
url: url,
success: function (httpResponse) {
var restaurants = [];
var json = httpResponse.data;
var venues = json.response.groups[0].items;
console.log(venues.length)
for(i = 0; i < venues.length; i++) {
venue = venues[i].venue;
var RestaurantObject = Parse.Object.extend("Restaurant");
var rest = new RestaurantObject();
try {
rest.set("geoLocation",
new Parse.GeoPoint({latitude: venue.location.lat,
longitude: venue.location.lng}));
} catch(err) {}
try {
rest.set("address", venue.location.address + " " + venue.location.formattedAddress[1]);
} catch(err) {}
try {
rest.set("phoneNumber", venue.contact.formattedPhone);
} catch(err) {}
try {
rest.set("website", venue.url);
} catch(err) {}
rest.set("name", venue.name);
rest.set("lowerName", venue.name.toLowerCase());
try {
rest.set("priceLevel", venue.price.tier);
} catch(err) {}
try {
rest.set("rating", venue.rating/2);
} catch(err) {}
try {
rest.set("storeId", venue.id);
} catch(err) {}
try {
rest.set("icon", venue.photos.groups[0].items[0].prefix + "original"
+ venue.photos.groups[0].items[0].suffix)
} catch(err) {}
restaurants.push(rest);
}
Parse.Object.saveAll(restaurants);
},
error: function (httpResponse) {
response.error("Request failed with response code:" + httpResponse.status + " Message: "
+ httpResponse.text);
}
});
});
I believe your issue is that you aren't returning the Promise from Parse.Object.saveAll(restaurants) when your httpRequest() is complete. Try returning that saveAll() promise and see if it completes.
I am a rookie in Nodejs and asynchronous programming. I am having a problem executing a GET request inside an asynchronous function. Here I am posting the whole code. I am trying to pull a list of all Urls , add them to a list and send the list for processing to another function.
My problem is with processing them. Inturn for each url I am executing a GET request to fetch the body and to look for image elements in it. I am looking to pass the Image url to a 3rd party api as a GET param. I am unable to execute the GET request as the control doesn't seem to reach there at all.
var async = require("async"),
request = require("request"),
cheerio = require("cheerio");
async.waterfall([
function(callback) {
var url = "someSourceUrl";
var linkList = [];
request(url, function(err, resp, body) {
var $ = cheerio.load(body);
$('.list_more li').each(function() {
//Find all urls and add them to a list
$(this).find('a').each(function() {
linkList.push($(this).attr('href'));
});
});
callback(null, linkList);
});
},
//pass all the links as a list to callback
function(liksListFetched, callback) {
for (var i in liksListFetched) {
callback(null, liksListFetched[i]);
}
}],
//***********My problem is with the below code**************
function(err, curUrl) {
var cuResp = "";
console.log("Currently Processing Url : " + curUrl);
request(curUrl, function(err, resp, body) {
var $ = cheerio.load(body);
var article = $("article");
var articleImage = article.find("figure").children('img').attr('src');
var responseGrabbed = "API response : ";
//check if there is an IMG element
if (articleImage === undefined) {
console.log("No Image Found.");
articleImage = 'none';
}
else {
//if there is an img element, pass this image url to an API,
//So do a GET call by passing imageUrl to the API as a GET param
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) { //code doesn't seem to reach here
I would like to grab the response and concatenate it to the responseGrabbed var.
console.log(resp);
responseGrabbed += resp;
});
}
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
process.exit(0);
});
});
I appreciate if any one can help me understand the root cause. Thanks in advance.
request() is asynchronous, so when you're console logging the string, the string hasn't been built yet, you have to do the console log inside the callback :
request("http://apiurl.tld?imageurl=" + articleImage, function(error, response, resp) {
responseGrabbed += resp;
console.log(responseGrabbed);// api response never gets concatenated :(
console.log("_=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=__=_=_=_=_=_=_");
});
Same goes for terminating the process, which should be done when all the requests have finished