Download functionality using streams for large files in NodeJS - javascript

I am trying to implement a download functionality using streams in NodeJS.
In the code I am trying to simulate an endpoint that sends data in chunks, something similar to paginated data, for example chunks in size of 5000. Or to make it further clear, we can send top and skip parameters to the endpoint to get a particular chunk of data. If no parameters are provided, it send the first 5000 entries.
There are 2 cases that I am trying to take care of:
When the user cancels the download from the browser, how do I handle the continuous fetching of data from the endpoint
When the user pauses the download from the browser, how do I pause the data fetching, and then resume once user resumes it
The first case can be taken care of using 'close' event of request. When the connection between the client and the server get cancelled, I disconnect.
If anyone has a better way of implementing this please suggest.
I am having trouble handling the second case when the user pauses.
If anyone could guide me through this, or even provide a better solution to the overall problem(incl. handling the chunks of data), it would be really helpful.
const {createServer} = require('http');
const {Transform} = require('stream');
const axios = require('axios');
var c = 0;
class ApiStream extends Transform {
constructor(apiCallback, res, req) {
super();
this.apiCallback = apiCallback;
this.isPipeSetup = false;
this.res = res;
this.req = req
}
//Will get data continuously
async start() {
let response;
try {
response = await this.apiCallback();
} catch (e) {
response = null;
}
if (!this.isPipeSetup) {
this.pipe(this.res);
this.isPipeSetup = true;
}
if (response) {
response = response.data
if (Array.isArray(response)) {
response.forEach((item) => {
this.push(JSON.stringify(item) + "\n");
});
} else if (typeof response === "object") {
this.push(JSON.stringify(response) + "\n");
} else if (typeof response === "string") {
this.push(response + "\n");
}
this.start()
}else{
this.push(null);
console.log('Stream ended')
}
}
}
const server = createServer(async (req, res, stream) => {
res.setHeader("Content-disposition", "attachment; filename=download.json");
res.setHeader("Content-type", "text/plain");
let disconnected = false;
const filestream = new ApiStream(async () => {
let response;
try {
if(disconnected){
console.log('Client connection closed')
return null;
}
c++;
response = await axios.get("https://jsonplaceholder.typicode.com/users");
//Simulate delay in data fetching
let z = 0;
if(c>=200) response = null;
while(z<10000){
let b = 0;
while(b<10000){
b+=0.5;
}
z +=0.5;
}
} catch (error) {
res.status(500).send(error);
}
if (response) {
return response;
}
return null;
}, res, req);
await filestream.start();
req.on('close', (err) => {
disconnected = true;
})
})
server.listen(5050, () => console.log('server running on port 5050'));

Related

Flutter PWA Service Worker Offline Setup

I have a Flutter App with a platform being on a PWA. Having some problems with getting the installability function on the Chrome browser to show up for it. It appears that Google has introduced a set of checks to the service worker, in particular the event listener for 'fetch' to see if it can handle a offline mode appropriately: https://developer.chrome.com/blog/improved-pwa-offline-detection/
I'm not entirely sure what needs to change on the bundled service worker that Flutter creates when building for Web, so I assume a custom implementation will be needed:
// The application shell files that are downloaded before a service worker can
// start.
const CORE = [
"/",
"main.dart.js",
"index.html",
"assets/NOTICES",
"assets/AssetManifest.json",
"assets/FontManifest.json"];
// During install, the TEMP cache is populated with the application shell files.
self.addEventListener("install", (event) => {
self.skipWaiting();
return event.waitUntil(
caches.open(TEMP).then((cache) => {
return cache.addAll(
CORE.map((value) => new Request(value + '?revision=' + RESOURCES[value], {'cache': 'reload'})));
})
);
});
// During activate, the cache is populated with the temp files downloaded in
// install. If this service worker is upgrading from one with a saved
// MANIFEST, then use this to retain unchanged resource files.
self.addEventListener("activate", function(event) {
return event.waitUntil(async function() {
try {
var contentCache = await caches.open(CACHE_NAME);
var tempCache = await caches.open(TEMP);
var manifestCache = await caches.open(MANIFEST);
var manifest = await manifestCache.match('manifest');
// When there is no prior manifest, clear the entire cache.
if (!manifest) {
await caches.delete(CACHE_NAME);
contentCache = await caches.open(CACHE_NAME);
for (var request of await tempCache.keys()) {
var response = await tempCache.match(request);
await contentCache.put(request, response);
}
await caches.delete(TEMP);
// Save the manifest to make future upgrades efficient.
await manifestCache.put('manifest', new Response(JSON.stringify(RESOURCES)));
return;
}
var oldManifest = await manifest.json();
var origin = self.location.origin;
for (var request of await contentCache.keys()) {
var key = request.url.substring(origin.length + 1);
if (key == "") {
key = "/";
}
// If a resource from the old manifest is not in the new cache, or if
// the MD5 sum has changed, delete it. Otherwise the resource is left
// in the cache and can be reused by the new service worker.
if (!RESOURCES[key] || RESOURCES[key] != oldManifest[key]) {
await contentCache.delete(request);
}
}
// Populate the cache with the app shell TEMP files, potentially overwriting
// cache files preserved above.
for (var request of await tempCache.keys()) {
var response = await tempCache.match(request);
await contentCache.put(request, response);
}
await caches.delete(TEMP);
// Save the manifest to make future upgrades efficient.
await manifestCache.put('manifest', new Response(JSON.stringify(RESOURCES)));
return;
} catch (err) {
// On an unhandled exception the state of the cache cannot be guaranteed.
console.error('Failed to upgrade service worker: ' + err);
await caches.delete(CACHE_NAME);
await caches.delete(TEMP);
await caches.delete(MANIFEST);
}
}());
});
// The fetch handler redirects requests for RESOURCE files to the service
// worker cache.
self.addEventListener("fetch", (event) => {
if (event.request.method !== 'GET') {
return;
}
var origin = self.location.origin;
var key = event.request.url.substring(origin.length + 1);
// Redirect URLs to the index.html
if (key.indexOf('?v=') != -1) {
key = key.split('?v=')[0];
}
if (event.request.url == origin || event.request.url.startsWith(origin + '/#') || key == '') {
key = '/';
}
// If the URL is not the RESOURCE list then return to signal that the
// browser should take over.
if (!RESOURCES[key]) {
return;
}
// If the URL is the index.html, perform an online-first request.
if (key == '/') {
return onlineFirst(event);
}
event.respondWith(caches.open(CACHE_NAME)
.then((cache) => {
return cache.match(event.request).then((response) => {
// Either respond with the cached resource, or perform a fetch and
// lazily populate the cache.
return response || fetch(event.request).then((response) => {
cache.put(event.request, response.clone());
return response;
});
})
})
);
});
self.addEventListener('message', (event) => {
// SkipWaiting can be used to immediately activate a waiting service worker.
// This will also require a page refresh triggered by the main worker.
if (event.data === 'skipWaiting') {
self.skipWaiting();
return;
}
if (event.data === 'downloadOffline') {
downloadOffline();
return;
}
});
// Download offline will check the RESOURCES for all files not in the cache
// and populate them.
async function downloadOffline() {
var resources = [];
var contentCache = await caches.open(CACHE_NAME);
var currentContent = {};
for (var request of await contentCache.keys()) {
var key = request.url.substring(origin.length + 1);
if (key == "") {
key = "/";
}
currentContent[key] = true;
}
for (var resourceKey in Object.keys(RESOURCES)) {
if (!currentContent[resourceKey]) {
resources.push(resourceKey);
}
}
return contentCache.addAll(resources);
}
// Attempt to download the resource online before falling back to
// the offline cache.
function onlineFirst(event) {
return event.respondWith(
fetch(event.request).then((response) => {
return caches.open(CACHE_NAME).then((cache) => {
cache.put(event.request, response.clone());
return response;
});
}).catch((error) => {
return caches.open(CACHE_NAME).then((cache) => {
return cache.match(event.request).then((response) => {
if (response != null) {
return response;
}
throw error;
});
});
})
);
}
I've peered into the following sources: https://javascript.plainenglish.io/your-pwa-is-going-to-break-in-august-2021-34982f329f40 and https://developer.mozilla.org/en-US/docs/Web/Progressive_web_apps/Offline_Service_workers, though I'm still receiving the same issue:
Chrome Application Tab - Manifest

UDP pinger timeout in javascript dgram node

So, for a course i'm taking, we're coding a UDP pinger in Javascript, using Node.js and Dgram. We've been given the following assignment:
Create the client code for an application. Your client should send 10 ping messages to the target UDP server. For each message, your client should calculate the round trip time from when the package is sent to when the response is received. Should a package be dropped along the way, the client is to handle this as well. This should be done by having the client wait 1 second for a response after sending each package. If no reply is received, the client should log accordingly (package lost, no response, timeout, etc.) and send a new package to try again. However, the total amount of packages sent should still only be 10. The client should also calculate a percentage of packages lost/no response received, and log this before connection is closed.
THis if course seems rather straight forward, and I thought so. I've been coding it for a while, and I'm almost finished, but I'm having issues with the aspect of making the client send a package, await response, and then act accordingly.
So far, what my code does is basically to send a ping, and when a pong is received, it sends another ping. What I can't figure out is how to make it log that a response wasn't received before sending the next package. In other words, I know how to make it react to a received response, I just don't know how to make it respond if no response is given within a set timeframe. I've tried playing around with if-statements and loops, as well as async functions, but I haven't made it work yet, so now I'm asking for help.
Code is here:
const dgram = require("dgram");
const ms = require("ms");
var client = dgram.createSocket("udp4");
const PORT = 8000;
const HOST = "localhost";
let today = "";
let t0 = "";
let t1 = "";
let RTT = "";
let sentPackages = "";
let receivedPackages = "";
const messageOutbound = Buffer.from("You Up?");
sendPackage();
const x = setInterval(sendPackage, 1000);
client.on("message", (message, remote) => {
receivedPackages++
today = new Date();
t1 = today.getTime();
console.log(
`Message from: ${remote.address}:${remote.port} saying: ${message}`
);
RTT = ms(t1 - t0, { long: true });
console.log(RTT);
const x = setInterval(sendPackage, 1000);
});
client.on('error', (err) => {
console.log(`server error:\n${err.stack}`);
server.close();
});
async function sendPackage() {
if (sentPackages < 10) {
client.send(messageOutbound, 0, messageOutbound.length, PORT, HOST, () => {
sentPackages++
let today = new Date();
t0 = today.getTime();
console.log(
`message has been sent to ${HOST}:${PORT}. Message sent at: ${t0}`
);
});
} else {
calculateLoss();
client.close();
}
};
function calculateLoss() {
let amountLost = sentPackages - receivedPackages;
let percentageLoss = amountLost / sentPackages * 100
console.log(amountLost);
console.log(percentageLoss +"% of packages lost");
};
I would use async / await to simply wait 1000ms / 1s between messages, then keep track of all messages in an array.
We identify messages with a uuid, so we can ensure that messages we receive can be matched to those we send.
We can then log all the required statistics afterwards:
const dgram = require("dgram");
const uuid = require('uuid');
const PORT = 8000;
const HOST = "localhost";
const client = dgram.createSocket("udp4");
// Array that keeps track of the messages we send
let messages = [];
// When we get a message, decode it and update our message list accordingly...
client.on("message", (messageBuffer, remote) => {
let receivedMessage = bufferToMessage(messageBuffer);
// Find the message we sent and set the response time accordingly.
let message = messages.find(message => message.uuid === (receivedMessage ||{}).uuid);
if (message) {
message.responseTimestamp = new Date().getTime();
}
});
client.on('error', (err) => {
console.log(`server error:\n${err.stack}`);
server.close();
});
function createMessage() {
return { uuid: uuid.v4() };
}
function messageToBuffer(message) {
return Buffer.from(JSON.stringify(message), "utf-8");
}
function bufferToMessage(buffer) {
try {
return JSON.parse(buffer.toString("utf-8"));
} catch (error) {
return null;
}
}
// Wait for timeout milliseconds
function wait(timeout) {
return new Promise(resolve => setTimeout(resolve, timeout));
}
function sendMessage(message, port, host) {
// Save the messages to our list...
messages.push(message);
console.log(`Sending message #${messages.length}...`);
// Set the time we send out message...
message.sentTimestamp = new Date().getTime();
let messageBuffer = messageToBuffer(message);
return new Promise((resolve, reject) => {
client.send(messageBuffer, 0, messageBuffer.length, port, host, (error, bytes) => {
if (error) {
reject(error);
} else {
resolve(bytes);
}
})
});
}
async function sendMessages(messageCount, port, host, timeout) {
for(let messageIndex = 0; messageIndex < messageCount; messageIndex++) {
let message = createMessage();
await sendMessage(message, port, host);
await wait(timeout);
if (message.responseTimestamp) {
console.log(`Response received after ${message.responseTimestamp - message.sentTimestamp} ms...`);
} else {
console.log(`No response received after ${timeout} ms...`);
}
}
logStatistics(messages);
}
function logStatistics(messages) {
let messagesSent = messages.length;
let messagesReceived = messages.filter(m => m.responseTimestamp).length;
let messagesLost = messagesSent - messagesReceived;
console.log(`Total messages sent: ${messagesSent}`);
console.log(`Total messages received: ${messagesReceived}`);
console.log(`Total messages lost: ${messagesLost} / ${(100*messagesLost / (messages.length || 1) ).toFixed(2)}%`);
if (messagesReceived > 0) {
console.log(`Average response interval:`, messages.filter(m => m.responseTimestamp).reduce((averageTime, message) => {
averageTime += (message.responseTimestamp - message.sentTimestamp) / messagesReceived;
return averageTime;
}, 0) + " ms");
}
}
sendMessages(10, PORT, HOST, 1000);

Client side is rendering faster than my data loading on server side nodeJS

I am trying to do a traffic analysis app but my client side is rendering faster than my data loading on the server side. (More specifically, it's the visual recognition data that is not being rendered on the web, I am using IBM visual recognition cloud service)
May I know how can I solve this issue? I have been stuck at this for a few days.
This is my code where I calling my function.
fetchAPI(10).then((data) => {
var data = data["items"][0]["cameras"]
for (let i = 0; i < data.length; i++){
image = data[i]["image"]
images.push(image);
coordinates.push([data[i]["location"]["latitude"], data[i]["location"]["longitude"]]);
}}).then(recognition(10, pageRender))
This is where I do my visual recognition
async function recognition(res, callback){
// Fetch api to get the images to be analyse
var data = await fetchAPI(10)
var info = await data["items"][0]["cameras"]
// Loop through the images
for (let i = 0; i < info.length; i++){
var image = await info[i]["image"]
// Params to be analyse
var params = await {
url: image,
classifier_ids: classifier_ids
};
// The part where it analyse the image
await visualRecognition.classify(params, function(err, response) {
if (err)
console.log(err);
else
try{
var status = (JSON.stringify(response.images[0]["classifiers"][0]["classes"][0]["class"], null, 2));
var prob = (JSON.stringify(response.images[0]["classifiers"][0]["classes"][0]["score"], null, 2));
}
catch(error){
status = "Error getting status"
prob = "Error getting probability"
}
value.push([status, prob]);
});
}
await callback()}
This is the callback function which I use to render my page.
function pageRender(){
app.get("/", (req,res) => {
res.render("index", {
images: JSON.stringify(images),
coordinates: JSON.stringify(coordinates),
value: JSON.stringify(value),
})
console.log("I have finish rendering")
}) }

Manage multiple Websocket client connection with node.js

I'm implementing a websocket client in node and my webhook's trying to handle multiple connections from a chatbot service. For example: a new user income, a websocket connection is established on an external chatbot service. The websocket URL is obtained through XMLHttpRequest in my code. And then I use this url to connect to the chatbot service using the ws object (new WebSocket('wssUrlObtainedThroughAjaxRequest','default-protocol')). So each user have a WebSocket. The question is that my code runs sequentially. So if two people sends message to my webhook node, things don't works properly because parallelism. Well, I'll post example code here to make it better to understand.
const express = require('express');
const PORT = process.env.PORT || 8002;
let WebSocket = require('ws');
let CONNECTIONS = new Map();
...
...
...
const app = express()
.use(bodyparser.urlencoded({extended: false}))
.use(bodyparser.json());
app.post('/', (req, res) => {
...
...
...
} else if (req.body.type === 'MESSAGE') {
let DM = req.body.space.name;
let msg = req.body.message.text;
ws = (CONNECTIONS.get(DM)!=null) ? CONNECTIONS.get(DM) : null;
if(ws==null || ws.readyState==3){
controlws.gerarURLWS();
ws = new WebSocket(controlws.urlws, 'talk-protocol');
CONNECTIONS.set(DM,ws);
}
// Executes on websocket openning
ws.onopen = function (event) {
console.log('Canal aberto;');
keepAliveWS();
ws.send(JSON.stringify(msgKoreAi(msg)));
}
if(ws.readyState==1)
ws.send(JSON.stringify(msgKoreAi(msg)));
ws.onmessage = async function (event) {
let resp = JSON.parse(event.data);
if (resp.type == "bot_response") {
text = resp.message[0].component.payload.text;
if(text==null){ // tem quick reply
//mount card hangouts response
let qreplies = resp.message[0].component.payload.payload.quick_replies;
card = '{"sections": [{"widgets": [{"buttons": [';
for(let i=0; i<qreplies.length; i++){
if(i!=qreplies.length-1)
card+='{"textButton": {"text": "'+qreplies[i].payload+'","onClick": {"action": {"actionMethodName": "'+qreplies[i].payload+'"}}}},';
else
card+='{"textButton": {"text": "'+qreplies[i].payload+'","onClick": {"action": {"actionMethodName": "'+qreplies[i].payload+'"}}}}';
}
card+=']}]}],"name": "respostas"}';
card = JSON.parse(card);
text = resp.message[0].component.payload.payload.text;
{
await assyncMessage(DM, text);
await assyncMessage(DM, card);
}
return;
}
//Send assync messages if synchronous was already sent
if(res.headersSent){
{
return await assyncMessage(DM, text);
}
}
else
return res.json({text});
}
}
return;
}
...
...
...
app.listen(PORT, () => {
console.log(`Server is running in port - ${PORT}`);
});

Running node-rdkafka code in server

I'm running the below node-rdkafka code in Eclipse as Node.js application. This is the sample code from https://blizzard.github.io/node-rdkafka/current/tutorial-producer_.html
I want to run this in a test server and call from iOS Mobile application.
I knew about running node.js app in AWS.
Question I: Is there any other options to run in a free test server environment like Tomcat?
Question II: Even If I am able to run this node.js app in a server, how do i call from a mobile application? Do I need to call producer.on('ready', function(arg) (or) What function i need to call from Mobile app?
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function() {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
/*
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});*/
//starting the producer
producer.connect();
First of all, you need an HTTP server. ExpressJS can be used. Then, just tack on the Express code basically at the end, but move the producer loop into the request route.
So, start with what you had
var Kafka = require('node-rdkafka');
//console.log(Kafka.features);
//console.log(Kafka.librdkafkaVersion);
var producer = new Kafka.Producer({
'metadata.broker.list': 'localhost:9092',
'dr_cb': true
});
var topicName = 'MyTest';
//logging debug messages, if debug is enabled
producer.on('event.log', function(log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function(err) {
console.error('Error from producer');
console.error(err);
});
producer.on('delivery-report', function(err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function(arg) {
console.log('producer ready.' + JSON.stringify(arg));
});
producer.on('disconnected', function(arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
Then, you can add this in the same file.
var express = require('express')
var app = express()
app.get('/', (req, res) => res.send('Ready to send messages!'))
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer('MyProducerTest - value-' +i);
var key = "key-"+i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
} // end if
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'))
I don't think the poll loop is necessary since you don't care about the counter anymore.
Now, connect your mobile app to http://<your server IP>:3000/ and send test messages with a POST request to http://<your server IP>:3000/10, for example, and adjust to change the number of messages to send
I might be late on this but this is how I did using promises and found it better than have a time out etc.
const postMessageToPublisher = (req, res) => {
return new Promise((resolve, reject) => {
producer.connect();
producer.setPollInterval(globalConfigs.producerPollingTime);
const actualBody = requestBody.data;
const requestBody = req.body;
const topicName = req.body.topicName;
const key = requestBody.key || uuid();
const partition = requestBody.partition || undefined;
const data = Buffer.from(JSON.stringify(udpatedBody));
/**
* Actual messages are sent here when the producer is ready
*/
producer.on(kafkaEvents.READY, () => {
try {
producer.produce(
topic,
partition,
message,
key // setting key user provided or UUID
);
} catch (error) {
reject(error);
}
});
// Register listener for debug information; only invoked if debug option set in driver_options
producer.on(kafkaEvents.LOG, log => {
logger.info('Producer event log notification for debugging:', log);
});
// Register error listener
producer.on(kafkaEvents.ERROR, err => {
logger.error('Error from producer:' + JSON.stringify(err));
reject(err);
});
// Register delivery report listener
producer.on(kafkaEvents.PUBLISH_ACKNOWLEDGMENT, (err, ackMessage) => {
if (err) {
logger.error(
'Delivery report: Failed sending message ' + ackMessage.value
);
logger.error('and the error is :', err);
reject({ value: ackMessage.value, error: err });
} else {
resolve({
teamName: globalConfigs.TeamNameService,
topicName: ackMessage.topic,
key: ackMessage.key.toString()
});
}
});
});
};
Please note that kafkaEvents contains my constants for the events we listen to and it is just a reference such as kafkaEvents.LOG is same as event.log
and also the calling function is expecting this to a promise and accordingly we user .then(data => 'send your response to user from here') and .catch(error => 'send error response to user
this is how I achieved it using promises

Categories