I am trying to create a script that pulls from the coin market cap API and displays the current price. The script is working fine on the back end when I assign the variable a value. However, when I try to run the function on sheets the returned value is null.
function marketview(ticker) {
var url = "https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?CMC_PRO_API_KEY=XXX&symbol=" + ticker;
var data = UrlFetchApp.fetch(url);
const jsondata = JSON.parse(data);
Logger.log(jsondata.data[ticker].quote['USD'].price)
}
My execution logs show that the scripts are running, but when when I use the function and try and quote ETH for example, the script is running for BTC.
When I do this on the backend and assign ETH the script works fine and returns the right quote. Any ideas on what I'm missing?
I did the same with coingecko API and add an issue having all my requests being rejected with quota exceeded error.
I understood that Google sheets servers IPs address were already spamming coingecko server. (I was obviously not the only one to try this).
This is why I used an external service like apify.com to pull the data and re-expose data over their API.
This is my AppScripts coingecko.gs:
/**
* get latest coingecko market prices dataset
*/
async function GET_COINGECKO_PRICES(key, actor) {
const coinGeckoUrl = `https://api.apify.com/v2/acts/${actor}/runs/last/dataset/items?token=${key}&status=SUCCEEDED`
return ImportJSON(coinGeckoUrl);
}
You need ImportJSON function, available here: https://github.com/bradjasper/ImportJSON/blob/master/ImportJSON.gs
Then in a cell I write: =GET_COINGECKO_PRICES(APIFY_API_KEY,APIFY_COINGECKO_MARKET_PRICES), you will have to create two field named APIFY_API_KEY and APIFY_COINGECKO_MARKET_PRICES in order for this to work.
Then register on apify.com, then you'll have to create an actor by forking apify-webscraper actor.
I set the StartURLs with https://api.coingecko.com/api/v3/coins/list, this will give me the total number of existing crypto (approx 11000 as of today), and number of page so I can run the request concurrently (rate limit is 10 concurrent requests on coingecko), then I just replace /list with /market and set the proper limit to get all the pages I need.
I use the following for the tasks page function:
async function pageFunction(context) {
let marketPrices = [];
const ENABLE_CONCURRENCY_BATCH = true;
const PRICE_CHANGE_PERCENTAGE = ['1h', '24h', '7d'];
const MAX_PAGE_TO_SCRAP = 10;
const MAX_PER_PAGE = 250;
const MAX_CONCURRENCY_BATCH_LIMIT = 10;
await context.WaitFor(5000);
const cryptoList = readJson();
const totalPage = Math.ceil(cryptoList.length / MAX_PER_PAGE);
context.log.info(`[Coingecko total cryptos count: ${cryptoList.length} (${totalPage} pages)]`)
function readJson() {
try {
const preEl = document.querySelector('body > pre');
return JSON.parse(preEl.innerText);
} catch (error) {
throw Error(`Failed to read JSON: ${error.message}`)
}
}
async function loadPage($page) {
try {
const params = {
vs_currency: 'usd',
page: $page,
per_page: MAX_PER_PAGE,
price_change_percentage: PRICE_CHANGE_PERCENTAGE.join(','),
sparkline: true,
}
let pageUrl = `${context.request.url.replace(/\/list$/, '/markets')}?`;
pageUrl += [
`vs_currency=${params.vs_currency}`,
`page=${params.page}`,
`per_page=${params.per_page}`,
`price_change_percentage=${params.price_change_percentage}`,
].join('&');
context.log.info(`GET page ${params.page} URL: ${pageUrl}`);
const page = await fetch(pageUrl).then((response) => response.json());
context.log.info(`Done GET page ${params.page} size ${page.length}`);
marketPrices = [...marketPrices, ...page];
return page
} catch (error) {
throw Error(`Fail to load page ${$page}: ${error.message}`)
}
}
try {
if (ENABLE_CONCURRENCY_BATCH) {
const fetchers = Array.from({ length: totalPage }).map((_, i) => {
const pageIndex = i + 1;
if (pageIndex > MAX_PAGE_TO_SCRAP) {
return null;
}
return () => loadPage(pageIndex);
}).filter(Boolean);
while (fetchers.length) {
await Promise.all(
fetchers.splice(0, MAX_CONCURRENCY_BATCH_LIMIT).map((f) => f())
);
}
} else {
let pageIndex = 1
let page = await loadPage(pageIndex)
while (page.length !== 0 && page <= MAX_PAGE_TO_SCRAP) {
pageIndex += 1
page = await loadPage(pageIndex)
}
}
} catch (error) {
context.log.info(`Fetchers failed: ${error.message}`);
}
context.log.info(`End: Updated ${marketPrices.length} prices for ${cryptoList.length} cryptos`);
const data = marketPrices.sort((a, b) => a.id.toLowerCase() > b.id.toLowerCase() ? 1 : -1);
context.log.info(JSON.stringify(data.find((item) => item.id.toLowerCase() === 'bitcoin')));
function sanitizer(item) {
item.symbol = item.symbol.toUpperCase()
return item;
}
return data.map(sanitizer)
}
I presume you are hiting the same issue I had with coinmarketcap, and that you could do the same with it.
You're not return ing anything to the sheet, but just logging it. Return it:
return jsondata.data[ticker].quote['USD'].price
Related
This question already has answers here:
How do I debug error ECONNRESET in Node.js?
(18 answers)
Closed 11 months ago.
I am using a node server to get trades data from binance. There are over a thousand pairs against which trades need to be fetched. The function takes time to run completely. I need the function to restart whenever it is finished running so I keep getting new data while my server is live and running. However, after my server has been running for 10-15 minutes, the following error occurs:
I want the server to run permanently in the background and for this function to keep fetching trades from API and storing those trades in my DB. I have another GET method defined that then fetches the trades from my DB.
The function that I am trying to run permanently lies in my main server.js file:
const getTrades = async () => {
let page = 1;
let coins = [];
const results = await db.query("SELECT * FROM pairs;");
const pairs = results.rows;
const latest = await db.query("SELECT MAX(trade_time) FROM trades");
const latestTrade = latest.rows[0].max;
while (page < 55) {
gecko = await axios(
`https://api.coingecko.com/api/v3/coins/markets?vs_currency=USD&order=market_cap_desc&per_page=250&page=${page}`
);
coins.push(gecko.data);
page++;
}
console.log("Loop over");
coins = coins.flat();
for (const pair of pairs) {
let biggestTrade = [];
response = await axios.get(
`https://api.binance.com/api/v3/trades?symbol=${pair.pair}`
);
let filtered = response.data;
filtered = filtered.filter((trade) => trade.time > latestTrade);
let sells = filtered.filter((trade) => trade.isBuyerMaker === true);
let buys = filtered.filter((trade) => trade.isBuyerMaker === false);
if (sells.length > 0) {
biggestTrade.push(
sells.reduce(function (prev, current) {
return prev.quoteQty > current.quoteQty ? prev : current;
})
);
}
if (buys.length > 0) {
biggestTrade.push(
buys.reduce(function (prev, current) {
return prev.quoteQty > current.quoteQty ? prev : current;
})
);
}
biggestTrade = biggestTrade.flat();
for (const trade of biggestTrade) {
let priceUSD = 0;
let baseAssetIcon = "";
for (const coin of coins) {
if (coin.symbol.toUpperCase() === pair.quote_asset) {
priceUSD = coin.current_price;
}
if (coin.symbol.toUpperCase() === pair.base_asset) {
baseAssetIcon = coin.image;
}
if (priceUSD > 0 && baseAssetIcon.length > 0) {
break;
}
}
if (trade.quoteQty * priceUSD > 50000) {
const results = db.query(
"INSERT INTO trades (exchange_name, exchange_icon_url, trade_time, price_in_quote_asset,price_in_usd, trade_value, base_asset_icon, qty, quoteQty, is_buyer_maker, pair, base_asset_trade, quote_asset_trade) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, $13)",
[
"Binance",
"https://assets.coingecko.com/markets/images/52/small/binance.jpg?1519353250",
trade.time,
trade.price,
priceUSD,
trade.quoteQty * priceUSD,
baseAssetIcon,
trade.qty,
trade.quoteQty,
trade.isBuyerMaker,
pair.pair,
pair.base_asset,
pair.quote_asset,
]
);
console.log("Trade Added");
}
}
}
console.log("PAIRS ARE OVER");
};
How can I make it so that the function runs repeatedly after a specified time period and the server does not break.
If you make continuous multiple calls to external third party API's without interval between calls, you are asking for being disconnected because API's have security policies that will prevents this kind of clients. Imagine if the entire world making 55 connections at once in a server. The server certainly will crash.
I see in your code you are making 55 calls at once. I recommend you put a delay between each call.
const delay = time => new Promise(res => setTimeout(res,time));
await delay(1000); // 1 second delay
There are other optimization that can prevent issues with connections in axios, like sharing httpAgent:
import http from "http"
import https from "https"
import axios from "axios"
const httpAgent = new http.Agent({ keepAlive: true })
const httpsAgent = new https.Agent({ keepAlive: true })
const api = axios.create({
baseURL: "http://google.com",
httpAgent,
httpsAgent,
})
//now you will reuse the axios instance:
while(page < 55) {
await delay(500);
gecko = await api(
`https://api.coingecko.com/api/v3/coins/markets?vs_currency=USD&order=market_cap_desc&per_page=250&page=${page}`
);
}
I want to implement a function in JavaScript which calls a series of web service endpoint and checks for a value in the response of the API call.
I need to achieve it in a way that the first endpoint page is called first then the there would be a filter method to filter out the specific object from the response. If the object is found, this process should break and the object must be returned. However if the object is not found in the first endpoint, then the second endpoint must be called and the same process is repeated until the object is found.
The Web service endpoint that I am working on is:
https://jsonmock.hackerrank.com/api/countries?page=1
This API returns a list of country data. Here the value of page query varies from 1 to 25. I need to call the endpoint and check for a specific country from 1 to 25 until the country object is found.
I tried achieving this using JavaScript Promise and Fetch API and couldn't think of a way to call the APIs one after the other.
I am really looking forward for your answer. Thank you in advance.
You can use async and await for this:
async function findCountry(country) {
for (let page = 1; page < 26; page++) {
console.log("page = " + page); // for debugging only
let response = await fetch("https://jsonmock.hackerrank.com/api/countries?page=" + page);
let {data} = await response.json();
let obj = data.find(obj => obj.name == country);
if (obj) return obj;
}
}
let country = "Belgium";
findCountry(country).then(obj => {
if (obj) {
console.log("The capital of " + country + " is " + obj.capital);
} else {
console.log("Could not find " + country);
}
});
If you know that the data is sorted by country name, then you could reduce the average number of requests by using a binary search.
Here's a way that you can do it.
const url = 'https://jsonmock.hackerrank.com/api/countries'
const fetchFromApi = async (countryName, page) => {
const res = await fetch(`${url}?page=${page}`)
return await res.json()
}
const getCountryFromResults = (countryName, data) => {
const country = countryName.toLowerCase()
return data.find(({name}) => name.toLowerCase() === country)
}
const findCountry = async (countryName) => {
let page = 1;
let totalPages = 1;
while(page <= totalPages) {
const res = await fetchFromApi(countryName, page);
if(totalPages < res.total_pages) {
totalPages = res.total_pages
}
const country = getCountryFromResults(countryName, res.data)
if(country){
return country
}
page = page + 1
}
}
( async () => {
console.log(await findCountry("Afghanistan"))
console.log(await findCountry("Argentina"))
}
)()
So I'm trying to connect to external server called Pexels to get some photos. I'm doing that from node.js but it is just a javascript issue. Pexels unfortunately lets user to download object with only 40 pictures per page.
https://api.pexels.com/v1/curated?per_page=40&page=1 // 40 is maximum
But actually I need more then that. I'd like to get 160 results, ie. to combine all first four pages. In order to do that I tried looping the request:
let pexelsData = [];
for(let i = 1; i < 5; i++) {
const randomPage = getRandomFromRange(1, 100); //pages should be randomized
const moreData = await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${randomPage}`,
createHeaders('bearer ', keys.pexelsKey));
pexelsData = [ ...moreData.data.photos, ...pexelsData ];
}
Now I can use pexelsData but it work very unstable, sometimes it is able to get all combined data, sometimes it crashes. Is there a correct and stable way of looping requests?
You work with 3rd party API, which has rate limits. So you should add rate limits to your code. The simplest solution for you is using p-limit or similar approach form promise-fun
It will looks like that:
const pLimit = require('p-limit');
const limit = pLimit(1);
const input = [
limit(() => fetchSomething('foo')),
limit(() => fetchSomething('bar')),
limit(() => doSomething())
];
(async () => {
// Only one promise is run at once
const result = await Promise.all(input);
console.log(result);
})();
you can break it into functions like..
let images=[];
const getResponse = async i=> {
if(i<5)
return await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${i}`)
}
const getImage = (i)=>{
if(i<5){
try {
const request = getResponse(i);
images = [...images,...request];
// here you will get all the images in an array
console.log(images)
getImage(++i)
} catch (error) {
console.log("catch error",error)
// getImage(i)
}
}
}
getImage(0); //call initail
I'm trying to create a simple example of payments over the XRPL using Ripple-lib. The idea is to send several payments to different accounts stored in an array. I've made it kind of work in a different way as it is expected, but when using the 'then' method (as the docs recommend) does not work at all.
I'm a total newbie to Javascript so I don't have a good grasp on the language nor asyncronous coding and promises. When using the 'then' paradigm, the code stops working and no output can be seen in the console. This is the code I'm currently using. In the comments inside the 'SendXRP' function I explain the problem. How can this be re-arranged? Between the two ways, what is the proper one to code it?
'use strict';
const RippleAPI = require('ripple-lib').RippleAPI;
const sender = 'r*********************************';
const secret = 's****************************';
const destinations = ['r*********************************',
'r*********************************',
'r*********************************'];
const amount = 5;
// Instantiate Ripple API
const api = new RippleAPI({
server: "wss://s.altnet.rippletest.net:51233"
});
run();
async function sendXRP(amount, fee, destination, memo) {
// Update amount
amount = (amount - fee).toString();
// Build payment
const payment = {
source: {
address: sender,
maxAmount: {
value: amount,
currency: 'XRP'
}
},
destination: {
address: destination,
amount: {
value: amount,
currency: 'XRP'
}
},
memos: [
{
data: memo
}
]
};
// Build instuctions
const instructions = {
maxLedgerVersionOffset: 5
};
console.log('Sending ' + amount + ' to ' + destination);
// THIS KIND OF WORKS FOR NOW
// Prepare the payment
const preparedTX = await api.preparePayment(sender, payment, instructions);
// Sign the payment
const signedTX = api.sign(preparedTX.txJSON, secret);
// Submit the payment
const result = await api.submit(signedTX['signedTransaction']);
// Return TX hash on successful TX
if ('resultCode' in result && result['resultCode'] == 'tesSUCCESS') {
return signedTX.id;
} else {
return null;
}
// THIS IS MORE SIMILAR TO HOW IT IS DONE IN THE DOCS! NOT WORKING!
// ALSO, HOW DO I RETURN THE RESULT OF API.SIGN TO THE MAIN FUNCTION?
// Prepare the payment
// api.preparePayment(sender, payment, instructions).then(preparedTX => {
// // Sign the payment
// api.sign(preparedTX.txJSON, secret).then(signedTX => {
// // Submit the payment
// api.submit(signedTX['signedTransaction']);
// })
// }).catch(console.error);
}
function run() {
// Connect to Ripple server
api.connect().then(() => {
return api.getFee();
}).then(async fee => {
for (var i in destinations) {
var hash = await sendXRP(amount, Number(fee), destinations[i], 'memotext');
console.log(hash);
}
}).then(() => {
return api.disconnect();
}).catch(console.error);
}
Could it be that some of the transactions failed to send? If it failed, the result variable from sendXRP should have the txresult, but since you returned null if the result code is not tesSUCCESS, it doesn't return the result information.
const result = await api.submit(signedTX['signedTransaction']);
if ('resultCode' in result && result['resultCode'] == 'tesSUCCESS') {
return signedTX.id;
} else {
return null;
}
Before, when I tried submitting transactions consecutively, it would fail and return error code tefPAST_SEQ.
"The sequence number of the transaction is lower than the current sequence number of the account sending the transaction." from https://developers.ripple.com/tef-codes.html
I recommend removing the if('resultCode' in result...) block and check the transaction result. If the transactions failed with tefPAST_SEQ error, my solution to this is set the account sequence in instructions manually or add setTimeOut after each submit.
I'm trying to get a random movie title from tmdb, the code is working but there's a frustrating problem, some random numbers return absolutely nothing, how can I loop or do something or try random numbers till i get a result? I guess there are gaps in tmdb movies id numbers!
const tmdb = require('tmdbv3').init('---');
function randomMovie(callback){
var r = Math.floor(Math.random()*1000);
tmdb.movie.info(r, (err ,res) => {
var x = res.title
callback(x);
})};
randomMovie(function(title){console.log(title)})
If you inspect the response headers and look at the status code I bet you're getting a 404 (not found) back since the ID has been deleted. And you won't be getting "nothing back" the TMDB API returns a set of errors with codes for you to troubleshoot what is happening. That is documented here.
Depending on what you're doing, knowing what IDs are available in advance will help you. There are some downloadable files for this that you can read about in the documentation.
I would not recommend to use this API, because even his test are failing. The problem you face is that if theres a number without data, it doesn't even return its callback, so you cannot handle its error (because it doesn't throw any). This is the best i could get from that API:
'use strict';
const tmdb = require('tmdbv3').init('8b39b6f141f42e463b507151122d0971');
function randomMovie(callback) {
const r = Math.floor(Math.random() * 1000);
tmdb.movie.info(r, (err, res) => {
const x = res.title
callback(x);
})
};
function tryTenTimes() {
let firstTitle;
for (let i = 0; i < 10; i++) {
randomMovie(title => {
if (!firstTitle) {
firstTitle = title;
console.log(firstTitle);
}
});
}
}
tryTenTimes();
It's an ugly workaround, that does 10 ask, and takes the first valid one.
EDIT: I was asked to do one with a while loop, and this is how I managed to do it
'use strict';
const tmdb = require('tmdbv3').init('8b39b6f141f42e463b507151122d0971');
function randomMovie(callback) {
const r = Math.floor(Math.random() * 1000);
tmdb.movie.info(r, (err, res) => {
const x = res.title
callback(x);
})
};
function tryWhileTimes() {
let firstTitle;
while (!firstTitle) {
return new Promise((resolve, reject) => {
randomMovie(title => {
if (!firstTitle) {
firstTitle = title;
console.log(firstTitle);
resolve();
}
});
});
}
}
tryWhileTimes()