I am trying to receive data via stdin which I have working:
const fs = require('fs');
const readStream = process.stdin;
readStream.pause();
readStream.setEncoding('utf8');
readStream.on('data', (data) => {
console.log(data);
});
readStream.resume();
Now what I need to do is store it as a variable so I can do some calculations before I return it via stdout.
Every time I try to so anything with it, like push certain data to an array it repeats the data until the stdin has finished, and I cant access it after it has finished. I cant find any resources online to help me.
You can use ReadableStream to process asynchronous tasks. Call .getReader() of ReadableStream instance to get an object which has a .read() method which when called returns an object having value and done properties. The controller passed to the constructor can queue tasks to be performed, the call to read() reads the enqueued data and sets the data at value property.
let n = 0;
let letters = "abcdefghijklmnopqrstuvwxyz";
const results = [];
let readableStream = new ReadableStream({
pull(controller) {
if (n < letters.length) controller.enqueue(letters[n++])
else controller.close();
}
});
let reader = readableStream.getReader();
let processStream = ({value, done}) => {
if (done) return reader.closed.then(() => results);
console.log(value);
// do stuff
let next = new Promise(resolve =>
setTimeout(() => {results.push(value); resolve()}
, Math.floor(Math.random() * 1200))
);
return next.then(() => reader.read())
.then(data => processStream(data));
}
reader.read()
.then(data => processStream(data))
.then(res => console.log(res));
You can also pipe the ReadableStream and .pipeTo() method of the instance to pipe the enqueued data to a WritableStream instance, where tasks can be performed at both read and write portions of the stream.
const letters = "abcdefghijklmnopqrstuvwxyz";
let RSController = class {
constructor(input) {
this.input = input;
this.n = 0;
}
async pull(controller) {
if (this.n < this.input.length) {
let curr = await this.handleData(this.input[this.n]);
++this.n;
controller.enqueue(curr);
} else {
this.n = 0;
controller.close();
}
}
cancel(err) {
console.log(err)
}
handleData(data) {
return new Promise(resolve => {
// do stuff
setTimeout(() => {
resolve(data)
}, Math.floor(Math.random() * 750))
})
}
}
let WSController = class {
constructor(arr = []) {
this.results = arr;
}
write(data) {
let dataUp = data.toUpperCase();
console.log(data, dataUp);
this.results.push([data, dataUp]);
}
close() {
console.log(JSON.stringify(this.results, null, 2));
}
abort(e) {
console.error(e);
}
}
let rscontroller = new RSController(letters);
let readableStream = new ReadableStream(rscontroller);
let wscontroller = new WSController([]);
let writableStream = new WritableStream(wscontroller);
readableStream
.pipeTo(writableStream);
Related
context: Two javascript classes in separate files, each integrating a different external service and being called in a express.js router.
See "problematic code" below:
route
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const data = await ( await ( await awsTranscribe.Upload(req.file, bucket)).CreateJob(transcribeParams)).GetJob()
res.send(data)
})
S3 class
class AmazonS3 {
constructor() {
this.Upload = this.Upload
}
async Upload(file, bucket) {
const uploadParams = {
Bucket: bucket,
Body: fs.createReadStream(file.path),
Key: file.filename,
}
this.data = await s3.upload(uploadParams).promise()
return this
}
}
Transcribe class
class Transcribe extends AwsS3 {
constructor() {
super()
this.CreateJob = this.CreateJob
this.GetJob = this.GetJob
}
async CreateJob(params) {
if(this.data?.Location) {
params.Media = { ...params.Media, MediaFileUri: this.data.Location }
}
this.data = await transcribeService.startTranscriptionJob(params).promise()
return this
}
async GetJob(jobName) {
if(this.data?.TranscriptionJob?.TranscriptionJobName) {
jobName = this.data.TranscriptionJob.TranscriptionJobName
}
this.data = await transcribeService.getTranscriptionJob({TranscriptionJobName: jobName}).promise()
return this
}
}
problem: the problem is with the chained awaits in the router file:
await ( await ( await awsTranscribe.Upload...
Yes, it does work, but it would be horrible for another person to maintain this code in the future.
How can i make so it would be just
awsTranscribe.Upload(req.file, bucket).CreateJob(transcribeParams).GetJob() without the .then?
The problem is with the chained awaits in the router file: await ( await ( await awsTranscribe.Upload...
No, that's fine. In particular it would be trivial to refactor it to separate lines:
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const a = await awsTranscribe.Upload(req.file, bucket);
const b = await b.CreateJob(transcribeParams);
const c = await b.GetJob();
res.send(c);
});
Your actual problem is that a, b, and c all refer to the same object awsTranscribe. Your code would also "work" if it was written
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
await awsTranscribe.Upload(req.file, bucket);
await awsTranscribe.CreateJob(transcribeParams);
await awsTranscribe.GetJob();
res.send(awsTranscribe);
});
The horrible thing is that you are passing your data between these methods through the mutable awsTranscribe.data property - even storing different kinds of data in it at different times! One could change the order of method calls and it would completely break in non-obvious and hard-to-debug ways.
Also it seems that multiple requests share the same awsTranscribe instance. This will not work with concurrent requests. Anything is possible from just "not working" to responding with the job data from a different user (request)! You absolutely need to fix that, then look at ugly syntax later.
What you really should do is get rid of the classes. There's no reason to use stateful objects here, this is plain procedural code. Write simple functions, taking parameters and returning values:
export async function uploadFile(file, bucket) {
const uploadParams = {
Bucket: bucket,
Body: fs.createReadStream(file.path),
Key: file.filename,
};
const data = s3.upload(uploadParams).promise();
return data.Location;
}
export async function createTranscriptionJob(location, params) {
params = {
...params,
Media: {
...params.Media,
MediaFileUri: location,
},
};
const data = await transcribeService.startTranscriptionJob(params).promise();
return data.TranscriptionJob;
}
async function getTranscriptionJob(job) {
const jobName = job.TranscriptionJobName;
return transcribeService.getTranscriptionJob({TranscriptionJobName: jobName}).promise();
}
Then you can import and call them as
routes.post('/aws', upload.single('file'), async (req, res) => {
const transcribeParams = JSON.parse(req.body.options)
const bucket = 'bucket-name'
const location = await uploadFile(req.file, bucket);
const job = await createTranscriptionJob(location, transcribeParams);
const data = await getTranscriptionJob(job);
res.send(c);
});
I got interested in whether it was possible to take an object with several async methods and somehow make them automatically chainable. Well, you can:
function chain(obj, methodsArray) {
if (!methodsArray || !methodsArray.length) {
throw new Error("methodsArray argument must be array of chainable method names");
}
const methods = new Set(methodsArray);
let lastPromise = Promise.resolve();
const proxy = new Proxy(obj, {
get(target, prop, receiver) {
if (prop === "_promise") {
return function() {
return lastPromise;
}
}
const val = Reflect.get(target, prop, receiver);
if (typeof val !== "function" || !methods.has(prop)) {
// no chaining if it's not a function
// or it's not listed as a chainable method
return val;
} else {
// return a stub function
return function(...args) {
// chain a function call
lastPromise = lastPromise.then(() => {
return val.apply(obj, args);
//return Reflect.apply(val, obj, ...args);
});
return proxy;
}
}
}
});
return proxy;
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
return delay(100);
}
}
const t = new Transcribe();
const obj = chain(t, ["getJob", "createJob"]);
log("begin");
obj.createJob().getJob()._promise().then(() => {
log("end");
});
There's a placeholder for your Transcribe class that has two asynchronous methods that return a promise.
Then, there's a chain() function that returns a proxy to an object that makes a set of passed in method names be chainable which allows you to then do something like this:
const t = new Transcribe();
// make chainable proxy
const obj = chain(t, ["getJob", "createJob"]);
obj.createJob().getJob()
or
await obj.createJob().getJob()._promise()
I wouldn't necessarily say this is production-ready code, but it is an interesting feasibility demonstration and (for me) a chance to learn more about a Javascript proxy object.
Here's a different approach that (instead of the proxy object) adds method stubs to a promise to make things chainable:
function chain(orig, methodsArray) {
let masterP = Promise.resolve();
function addMethods(dest) {
for (const m of methodsArray) {
dest[m] = function(...args) {
// chain onto master promise to force sequencing
masterP = masterP.then(result => {
return orig[m].apply(orig, ...args);
});
// add methods to the latest promise befor returning it
addMethods(masterP);
return masterP;
}
}
}
// add method to our returned promise
addMethods(masterP);
return masterP;
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
this.cntr = 0;
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
++this.cntr;
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
++this.cntr;
return delay(100);
}
}
const t = new Transcribe();
log("begin");
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
log("end");
});
Since this returns an actual promise (with additional methods attached), you can directly use .then() or await with it without the separate ._promise() that the first implementation required.
So, you can now do something like this:
const t = new Transcribe();
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
});
or:
const t = new Transcribe();
await chain(t, ["getJob", "createJob"]).createJob().getJob();
log(`cntr = ${t.cntr}`);
And, here's a third version where it creates a thenable object (a pseudo-promise) with the added methods on it (if it bothers you to add methods to an existing promise):
function chain(orig, methodsArray) {
if (!methodsArray || !methodsArray.length) {
throw new Error("methodsArray argument must be array of chainable method names");
}
let masterP = Promise.resolve();
function makeThenable() {
let obj = {};
for (const m of methodsArray) {
obj[m] = function(...args) {
// chain onto master promise to force sequencing
masterP = masterP.then(result => {
return orig[m].apply(orig, ...args);
});
return makeThenable();
}
}
obj.then = function(onFulfill, onReject) {
return masterP.then(onFulfill, onReject);
}
obj.catch = function(onReject) {
return masterP.catch(onReject);
}
obj.finally = function(onFinally) {
return masterP.finally(onFinally);
}
return obj;
}
return makeThenable();
}
function delay(t) {
return new Promise(resolve => {
setTimeout(resolve, t);
});
}
function log(...args) {
if (!log.start) {
log.start = Date.now();
}
const delta = Date.now() - log.start;
const deltaPad = (delta + "").padStart(6, "0");
console.log(`${deltaPad}: `, ...args)
}
class Transcribe {
constructor() {
this.greeting = "Hello";
this.cntr = 0;
}
async createJob(params) {
log(`createJob: ${this.greeting}`);
++this.cntr;
return delay(200);
}
async getJob(jobName) {
log(`getJob: ${this.greeting}`);
++this.cntr;
return delay(100);
}
}
const t = new Transcribe();
log("begin");
chain(t, ["getJob", "createJob"]).createJob().getJob().then(() => {
log(`cntr = ${t.cntr}`);
log("end");
});
I am trying to execute allCountryData and return a promise its working fine but after allCountryData is done executing I want to perform a operation on that returned data / or allCountryDataArray and store the highest values in arrayOfHighestCases
Note I can't chain the other login in allCountryData.
Please help let me know if you need any more details
export const allCountryDataArray = [];
export const arrayOfHighestCases = [];
const allCountryData = async () => {
sendHTTP()
.then((res) => {
return res.response;
})
.then((res) => {
allCountryDataArray.push(...res);
return allCountryDataArray;
});
return await allCountryDataArray;
// Highest Cases
};
The code is below is not working
const highestCasesData = async () => {
// const allCountryDataArrayy = await allCountryData();
// allCountryData()
// .then((data) => {
// console.log(arrayOfHighestCases[0]);
// })
// .then((res) => {
const np = new Promise((res, rej) => {
res(allCountryData());
});
return np.then((res) => {
console.log(res);
const arrayofHigh = allCountryDataArray.sort((a, b) => {
if (a.cases.total < b.cases.total) {
return 1;
} else if (a.cases.total > b.cases.total) {
return -1;
} else {
return 0;
}
});
console.log(arrayofHigh);
const slicedArray = arrayofHigh.slice(0, 6);
for (const eachHighCase of slicedArray) {
arrayOfHighestCases.push(eachHighCase);
}
console.log(arrayOfHighestCases);
return arrayOfHighestCases;
});
// });
};
highestCasesData();
Filling global arrays with async data is a way into timing conflicts. Bugs where the data ain't there, except when you look it is there and yet another question here on my SO about "Why can't my code access data? When I check in the console everything looks fine, but my code ain't working."
If you want to store something, store Promises of these arrays or memoize the functions.
const allCountryData = async () => {
const res = await sendHTTP();
return res.response;
};
const highestCasesData = async () => {
const allCountryDataArray = await allCountryData();
return allCountryDataArray
.slice() // make a copy, don't mutate the original array
.sort((a, b) => b.cases.total - a.cases.total) // sort it by total cases DESC
.slice(0, 6); // take the first 6 items with the highest total cases
}
This is working please let me know if I can make some more improvements
const allCountryData = async () => {
return sendHTTP()
.then((res) => {
return res.response;
})
.then((res) => {
allCountryDataArray.push(...res);
return allCountryDataArray;
});
// Highest Cases
};
const highestCasesData = async () => {
return allCountryData().then((res) => {
console.log(res);
const arrayofHigh = allCountryDataArray.sort((a, b) => {
if (a.cases.total < b.cases.total) {
return 1;
} else if (a.cases.total > b.cases.total) {
return -1;
} else {
return 0;
}
});
console.log(arrayofHigh);
const slicedArray = arrayofHigh.slice(0, 6);
for (const eachHighCase of slicedArray) {
arrayOfHighestCases.push(eachHighCase);
}
console.log(arrayOfHighestCases);
return arrayOfHighestCases;
});
};
highestCasesData();
I am not able to access the returned object property, Please tell me why its returning undefined when data is object and giving correct value.
This is function created to sendHTTPRequest based on data.
import { countryCap } from "./capitalizingFunc.js";
export const sendHTTPRequest = (country) => {
const capitalisedCountry = countryCap(country);
return fetch(
`https://covid-19-coronavirus-statistics.p.rapidapi.com/v1/total?country=${capitalisedCountry}`,
{
method: "GET",
headers: {
"x-rapidapi-key": "3b0f2e00ebmsh95246403d9540c9p1506d4jsn3c44ce26f745",
"x-rapidapi-host": "covid-19-coronavirus-statistics.p.rapidapi.com",
},
}
)
.then((response) => {
const newResponce = response.json();
return newResponce;
})
.catch((err) => {
console.error(err);
});
};
This is constructor class
export class casesDataFetcher {
constructor(countryName) {
sendHTTPRequest(countryName)
.then((response) => {
return response.data;
})
.then((data) => {
this.country = data.location;
this.cases = data.confirmed;
this.recovered = data.recovered;
this.deaths = data.deaths;
console.log(this);
return this;
});
}
}
This is execution function
import { casesDataFetcher } from "./casesDataFetcher.js";
export const screenDataShower = (country) => {
const dataStorage = [];
const globalInfected = document.querySelector(".infected h2");
const globalActive = document.querySelector(".active h2");
const globalDeaths = document.querySelector(".deaths h2");
const globalRecovered = document.querySelector(".recovered h2");
const globalCountries = document.querySelector(".countries h2");
let promise = new Promise(function (resolve, reject) {
const recordedData = new casesDataFetcher(country);
console.log(recordedData);
resolve(recordedData);
});
return promise.then((data) => {
console.log(typeof data);
console.log(typeof data);
console.log(data.cases); // NOT WORKING GIVING UNDEFINED
globalInfected.textContent = `${nn.cases}`;
globalActive.textContent = data.cases - data.recovered - data.deaths;
globalDeaths.textContent = data.deaths;
globalRecovered.textContent = data.recovered;
globalCountries.textContent = 219;
});
};
I also tried to convert the data to JSON again but still I was not able to access the property of returned data in screenDataShower
you're calling sendHTTPRequest inside casesDataFetcher's constructor, from your code there's no guarantee data is resolved when you access it
extract sendHTTPRequest into a new function and wrap into a promise
export class casesDataFetcher {
constructor(countryName) {
this.countryName = countryName
}
fetch = () => {
return new Promise((res, rej) => {
sendHTTPRequest(this.countryName)
.then((response) => {
return response.data;
})
.then((data) => {
this.country = data.location;
this.cases = data.confirmed;
this.recovered = data.recovered;
this.deaths = data.deaths;
console.log(this);
res(this);
});
})
}
}
make screenDataShower function async then you can await data from fetch function in casesDataFetcher, this way it can guarantee data is there when you access it
import { casesDataFetcher } from "./casesDataFetcher.js";
export const screenDataShower = async (country) => {
const dataStorage = [];
const globalInfected = document.querySelector(".infected h2");
const globalActive = document.querySelector(".active h2");
const globalDeaths = document.querySelector(".deaths h2");
const globalRecovered = document.querySelector(".recovered h2");
const globalCountries = document.querySelector(".countries h2");
const _casesDataFetcher = new casesDataFetcher(country)
const data = await _casesDataFetcher.fetch()
console.log(typeof data);
console.log(typeof data);
console.log(data.cases); // NOT WORKING GIVING UNDEFINED
globalInfected.textContent = `${nn.cases}`;
globalActive.textContent = data.cases - data.recovered - data.deaths;
globalDeaths.textContent = data.deaths;
globalRecovered.textContent = data.recovered;
globalCountries.textContent = 219;
};
The problem is that the json method of your response returns a promise instead of plain JSON. So you should change the call of the json method in your sendHTTPRequest function to something like:
.then((response) => {
const newResponse = response.json().then((jsonResponse) => jsonResponse);
return newResponse;
})
I'm new to node.js and javascript in general but I am having issues understanding why the writeFile function is writing a blank file. I think the for loop should be a Promise but I am not sure how to write it.
const removeProviders = function () {
readline.question('Where is the file located? ', function(filePath) {
let providerArray = fs.readFileSync(filePath).toString().split('\r\n');
console.log(providerArray);
let importFile = '';
for (let providerId in providerArray) {
getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
})
}
fs.writeFile('C:/path/to/file.txt', importFile);
You can collect all the promises from the for-loop into an Array and then pass them into Promise.all() which will resolve only after all of them resolved. If one of the promises are rejected, it will reject as well:
const promises = providerArray.map((item) => {
return getProvider(item)
.then((response) => {
let providerInfo = response.data;
return providerInfo;
})
.then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
});
});
Promise.all(promises).then(() => {
fs.writeFile('C:/path/to/file.txt', importFile, err => {
if (err) {
console.error(err);
}
});
});
While doing this you could also get rid of the importFile variable and collect directly the results of your promises. Promise.all().then(results => {}) will then give you an array of all results. Thus no need for an updatable variable.
The below approach may be useful.
I don't know your getProvider return Promise. you can set promise in getProvider method
const getProviderValue = async function(providerArray) {
return new Promise((resolve, reject) {
let importFile = '';
for (let providerId in providerArray) {
await getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += "entry";
})
}
resolve(importFile)
})
}
const removeProviders = async function () {
readline.question('Where is the file located? ', function(filePath) {
let providerArray = fs.readFileSync(filePath).toString().split('\r\n');
console.log(providerArray);
let importFile = await getProviderValue(providerArray)
fs.writeFile('C:/path/to/file.txt', importFile);
})
}
Your for loop does not wait for the promises to resolve. A better way to approach this problem would be to use reduce.
providerArray.reduce(
(p, _, i) => {
p.then(_ => new Promise(resolve =>
getProvider(providerArray[providerId]).then((response) => {
let providerInfo = response.data;
return providerInfo;
}).then((providerInfo) => {
let entry = createImportEntry(providerInfo);
importFile += entry;
resolve();
}))
);
}
, Promise.resolve() );
You can also use Promise.all but the out data may not be in the order that you expect if you append to the importFile variable.
I want to retrieve different HTML body at once and as soon as all results are available work with that content.
My callback solution which works looks like this (probably only relevant to read if the idea is not clear, otherwise skip ;)):
const request = require('request')
const argLength = process.argv.length
const result_array = []
let counter = 0
function executeRequest () {
for (start = 2; start <= argLength - 1; start++) {
const copy = start
function callback (error, res, body) {
const startCopy = copy
if (error) {
console.log('error')
return
}
result_array[startCopy - 2] = body.toString().length
counter++
if (counter === argLength - 2) {
console.log(result_array)
}
}
request(process.argv[start], callback)
}
}
executeRequest()
Now I try to make it running with Promises like this:
const httpRequest = require('request')
const argumentLength = process.argv.length
function fillURLArray () {
resultArray = []
for (start = 2; start < argumentLength; start++) {
resultArray[start - 2] = process.argv[start]
}
return resultArray
}
const urls = fillURLArray()
let counter = 0
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => httpRequest(url, (error, res, body) => {
console.log(body.toString())
resultArray[counter++] = body.toString()
}))).then(value => {
console.log('promise counter: ' + counter++)
console.log(resultArray)
console.log('called')
})
}
readHttp()
I tried already several attempts with different promise chains but every time I get either not a result or just an empty array. So obviously the .then() function is called before the array is actually filled (at least I guess so since console.log(body.toString()) appears to print the content some time later)
Any idea how to solve this with promises?
Thank you
request is not returning promise object so have created a method that return promise object on which you do Promise.all.
function requestPromise(url){
return new Promise((resovle,reject) => {
httpRequest(url, (error, res, body) => {
if(err){
reject(err);
}
resolve(body.toString());
});
});
}
function readHttp () {
const resultArray = []
Promise.all(urls.map(url => requestPromise(url))).then(values => {
console.log("counter => ",values.length);
resultArray = resultArray.concat(values);
console.log("values=> ",values);
console.log("resultArray=> ",resultArray);
});
}
httpRequest does not return a promise so you have to make one yourself, also your resultArray is not necessary:
const makeRequest = url => new Promise((resolve, reject) => httpRequest(url, (error, res) => error ? reject(error) : resolve(res)));
Promise.all(urls.map(makeRequest))
.then(result => {
console.log(result.map(res => res.body.toString()));
console.log('called');
});