We Keep Coding

How output is different for similar code-block in javascript

Why output is different in following:
1st scenario: prints
https://appmagic.io/modern/1
https://appmagic.io/modern/1
let urlHash = {};
const rootURL = 'http://tinyurl.com/';
var encode = function(longUrl) {
let hash = Date.now().toString(36);
urlHash[hash] = longUrl;
return `${rootURL}${hash}`
};
var decode = function(shortUrl) {
return urlHash[shortUrl.substring(rootURL.length)]
};
let url1 = encode("https://appmagic.io/classic/1");
let url2 = encode("https://appmagic.io/modern/1");
console.log(decode(url1));
console.log(decode(url2));
2nd scenario: prints
https://appmagic.io/classic/1
https://appmagic.io/modern/1
let urlHash = {};
const rootURL = 'http://tinyurl.com/';
var encode = function(longUrl) {
let hash = Date.now().toString(36);
console.log({hash}); // difference in code
console.log({hash}); // difference in code
urlHash[hash] = longUrl;
return `${rootURL}${hash}`
};
var decode = function(shortUrl) {
return urlHash[shortUrl.substring(rootURL.length)]
};
let url1 = encode("https://appmagic.io/classic/1");
let url2 = encode("https://appmagic.io/modern/1");
console.log(decode(url1));
console.log(decode(url2));
My guess is:
Since Date.now() gives values in milisecond, without console (IO operation i.e. time consuming sync operations) they get evaluate in nano-second and hash remains same, So the similar output in 1st scenario
But if we are adding console (IO operation i.e. time consuming sync operations) it delay operation for more than millisecond and different output comes in 2nd scenario.
I'm not sure if my perception is correct. Can any-one provide better/correct explanation.
If my guess is right, how can I create collision free fast hash,
thinking to use window.performance.now() but it is also not available in all browsers

While the comments address your main concern that it's possible to generate the same hash due to the encode function running more than once in the same millisecond, I'd like to leave this here as an example of addressing that by deferring the hashing until a unique key is generated:
function Coder(rootUrl) {
const urlHash = {}
return {
encode(longUrl) {
let hash
do {
hash = Date.now().toString(36)
} while (urlHash[hash])
urlHash[hash] = longUrl
return `${rootUrl}${hash}`
},
decode(shortUrl) {
return urlHash[shortUrl.substring(rootUrl.length)]
}
}
}
// usage example
const { decode, encode } = Coder('http://tinyurl.com/')
const url1 = encode('https://appmagic.io/classic/1')
const url2 = encode('https://appmagic.io/modern/1')
console.log('url1 encoded', url1)
console.log('url2 encoded', url2)
console.log('url1 decoded', decode(url1))
console.log('url2 decoded', decode(url2))
With this you'd only be able to generate one hash per millisecond, but I suppose that isn't a bad trade-off.

Fetching data in the loop

So I'm trying to connect to external server called Pexels to get some photos. I'm doing that from node.js but it is just a javascript issue. Pexels unfortunately lets user to download object with only 40 pictures per page.
https://api.pexels.com/v1/curated?per_page=40&page=1 // 40 is maximum
But actually I need more then that. I'd like to get 160 results, ie. to combine all first four pages. In order to do that I tried looping the request:
let pexelsData = [];
for(let i = 1; i < 5; i++) {
const randomPage = getRandomFromRange(1, 100); //pages should be randomized
const moreData = await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${randomPage}`,
createHeaders('bearer ', keys.pexelsKey));
pexelsData = [ ...moreData.data.photos, ...pexelsData ];
}
Now I can use pexelsData but it work very unstable, sometimes it is able to get all combined data, sometimes it crashes. Is there a correct and stable way of looping requests?

You work with 3rd party API, which has rate limits. So you should add rate limits to your code. The simplest solution for you is using p-limit or similar approach form promise-fun
It will looks like that:
const pLimit = require('p-limit');
const limit = pLimit(1);
const input = [
limit(() => fetchSomething('foo')),
limit(() => fetchSomething('bar')),
limit(() => doSomething())
];
(async () => {
// Only one promise is run at once
const result = await Promise.all(input);
console.log(result);
})();

you can break it into functions like..
let images=[];
const getResponse = async i=> {
if(i<5)
return await axios.get(`https://api.pexels.com/v1/curated?per_page=40&page=${i}`)
}
const getImage = (i)=>{
if(i<5){
try {
const request = getResponse(i);
images = [...images,...request];
// here you will get all the images in an array
console.log(images)
getImage(++i)
} catch (error) {
console.log("catch error",error)
// getImage(i)
}
}
}
getImage(0); //call initail

How can I get the raw download size of a request using Puppeteer?

That is, the total amount of data downloaded across all resources (including video/media), similar to that returned by Chrome DevTools' Network tab.

There doesn't seem to be any way to do this as of January 2018 that works with all resource types (listening for the response event fails for videos), and that correctly counts compressed resources.
The best workaround seems to be to listen for the Network.dataReceived event, and process the event manually:
const resources = {};
page._client.on('Network.dataReceived', (event) => {
const request = page._networkManager._requestIdToRequest.get(
event.requestId
);
if (request && request.url().startsWith('data:')) {
return;
}
const url = request.url();
// encodedDataLength is supposed to be the amount of data received
// over the wire, but it's often 0, so just use dataLength for consistency.
// https://chromedevtools.github.io/devtools-protocol/tot/Network/#event-dataReceived
// const length = event.encodedDataLength > 0 ?
// event.encodedDataLength : event.dataLength;
const length = event.dataLength;
if (url in resources) {
resources[url] += length;
} else {
resources[url] = length;
}
});
// page.goto(...), etc.
// totalCompressedBytes is unavailable; see comment above
const totalUncompressedBytes = Object.values(resources).reduce((a, n) => a + n, 0);

The solution of #mjs works perfectly even in 2021. Just need to replace:
page._networkManager -> page._frameManager._networkManager
Full example that works for me:
const resources = {};
page._client.on('Network.dataReceived', (event) => {
const request = page._frameManager._networkManager._requestIdToRequest.get(
event.requestId
);
if (request && request.url().startsWith('data:')) {
return;
}
const url = request.url();
const length = event.dataLength;
if (url in resources) {
resources[url] += length;
} else {
resources[url] = length;
}
});
await page.goto('https://stackoverflow.com/questions/48263345/how-can-i-get-the-raw-download-size-of-a-request-using-puppeteer');
const totalUncompressedBytes = Object.values(resources).reduce((a, n) => a + n, 0);
console.log(totalUncompressedBytes);

If you are using puppeteer, you have server side node... Why not pipe the request through a stream, or streams and then calculate the content size?
Also there is https://github.com/watson/request-stats
Also you may want to call page.waitForNavigation as you may be wrestling with async timing issues

const imgaes_width = await page.$$eval('img', anchors => [].map.call(anchors, img => img.width));
const imgaes_height = await page.$$eval('img', anchors => [].map.call(anchors, img => img.height));

Using an online tool to generate a random name

I found this website http://www.mess.be/inickgenwuname.php
It allows you to type in a name and it will generate a random rapper name. I wanted to have a button on my website that just generates the name for you so I decided to write some javascript that will send a request to this website and parse the response to get the random name.
Here is the node.js code I wrote.
function getRandomName() {
var http = require('http');
var data = {
realname:"something"
};
var querystring = require("querystring");
var qs = querystring.stringify(data);
var qslength = qs.length;
var options = {
hostname: "www.mess.be",
path: "/inickgenwuname.php",
method: 'POST',
headers:{
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': qslength
}
};
var str = "";
var req = http.request(options, function(res) {
res.on('data', function (chunk) {
str+=chunk;
});
res.on('end', function() {
var s = str.slice(str.indexOf("From this day forward, I will be known as... ") + "From this day forward, I will be known as... ".length,
str.indexOf("-And you"));
s = s.replace("\n", "").trim();
console.log(s);
});
});
req.write(qs);
req.end();
}
When I went to the website and pressed f12 on chrome and inspected the code, I found this little segment...
So this is what I used to formulate the request to the php. However, I only guessed through trial and error that the data that needed to be send was key-value pair object where the key is realname. My question is, how would I have known this otherwise? Is there no way to find out from the website, where the data being send with the POST is being received from?

Why by guessing? The form tells you everything that needs to be sent.
Also you could press F12 -> Network, and then send the request. After this you look at the sent requests and search for a POST request. When you click on the /inickgenwuname.php request you get more information about it. In there you can see Response Headers / Request Headers and as a last category "Form Data". There you can see all the data that is sent with this form.
I hope this is the answer you were looking for.

Stealing bandwidth without proper compensation (so called web-scraping) is quite commonly frowned upon. I couldn't find anything on that site that allows for it although I did not search thoroughly.
Why don't you roll your own? It's very simple, as can be seen in this Q&D hack:
function wu_names(input){
// some nice, fitting adjectives. Add more
var adjectives = ["annoying", "crazy", "expert", "insane", "lucky", "sardonic", "pestering"];
// some nice, fitting nouns. Add more
var nouns = ["assassin", "bastard", "conjurer", "destroyer", "ninja", "prophet", "wizard"];
var first = "";
var second = "";
var hash = 0;
var primitive_hash = function(s){
var h = 0;
for(var i = 0;i < s.length;i++){
var c = s.charCodeAt(i);
// standard hash = hash * 31 + c
h = ((h << 5) - h>>>0) + c;
}
return h;
};
first = input.split(" ")[0];
// no useful entry at all
if(first === undefined){
return null;
}
hash = primitive_hash(first);
first = adjectives[hash % adjectives.length];
second = input.split(" ")[1];
// no second entry
if(second === undefined){
return null;
}
hash = primitive_hash(second);
second = nouns[hash % nouns.length];
return first + " " + second;
}
The lists of adjectives and nouns is quite short, you might add to them, as the comments suggest.

Node/Javascript only send changed values

I'm writing a simple application where I send values to a mqtt broker given by a pot-meter (variable resistor). The thing I am trying to accomplish is that I only send changed values to save bandwidth. I am trying Object.observe, but that does not do anything. Can anybody help me?
My code:
var analogValue = 0;
every((0.5).second(), function() {
analogValue = my.sensor.analogRead();
var values = {values:[{key:'resistance', value: analogValue}]}
//another experiment here
var arr = ['resitance', analogValue];
Array.observe(arr, function(changes) {
console.log(changes);
});
arr[1] = analogValue
console.log('sent ',values,'to ',thingTopic)
client.publish(thingTopic, JSON.stringify(values));
});
var o = [analogValue];
Object.observe(o, function (changes) {
console.log(changes);
//eventually publish only changes to broker here
})
o.name = [analogValue]

You don't need to use Object.observe. You can just save the last measurement and check the new one against it. Like this:
// I'm assuming that any actual measurement will be different than 0
var lastMeasurement = 0;
every((0.5).second(), function() {
var analogValue = my.sensor.analogRead();
if (lastMeasurement !== analogValue) {
// the new value is different
var values = {values:[{key:'resistance', value: analogValue}]};
client.publish(thingTopic, JSON.stringify(values));
// update the last measurement value
lastMeasurement = analogValue;
}
});