I'm using javascript because I have no choice : the api that i'm using is in javascript and only in javascript. I've no knowledge about it at all. I'm trying to do a quit function that store data (polls list) in case in of an error before exiting. In parallel of that I have an init function that restore the data.
function quit() {
fs.readFile(pollsName, "utf8", function readFileCallback(err, data) {
// never reach this line
if (err) {
console.log(err);
} else {
obj = JSON.parse(data); //now it an object
obj.poll = polls.map((poll) => poll.serialize);
json = JSON.stringify(obj); //convert it back to json
fs.writeFile(jsonName, json, (e) => {
if (e) throw e;
console.log("Polls saved");
}); // write it back
}
});
exit(1);
}
As you can see I never enter in the readFile() function. However I have another function write() that correctly store logs in similar file. The file name is correct, it is a json file with a key "poll" that represent an empty list. Both fs and exit package are import with "require".
I currently have no clue to solve this situation and i'm struggling on it. At first I was handling signal but I realised it wasn't possible because write/read are unsafe. With this solution I'm going to catch every error and start this function.
Thank you if you can help me to find a solution
You call readFile, but then you immediately terminate the porgram using exit(1). Simply move the exit(1) into the callback function readFileCallback, so your program terminates after the polls are saved.
Related
I'm trying to use fluent-ffmpeg's ffprobe to get the metadata of a file and add it to a list, but I want to separate the process of getting the metadata from the method related to checking the file, mostly because the addFileToList() function is quite long as is and the ffprobe routine is quite long as well.
I've tried the following code, but it doesn't give the results I'm expecting:
export default {
// ...
methods: {
getVideoMetadata (file) {
const ffmpeg = require('fluent-ffmpeg')
ffmpeg.ffprobe(file.name, (err, metadata) => {
if (!err) {
console.log(metadata) // this shows the metadata just fine
return metadata
}
})
},
addFileToList (file) {
// file checking routines
console.log(this.getVideoMetadata(file)) // this returns null
item.metadata = this.getVideoMetadata(file)
// item saving routines
}
}
}
I've already tried to nest the getVideoMetadata() routines inside addFileToList(), and it works, but not as intended, because the actions are carried, but not the first time, only the second time. It seems to be an async issue, but I don't know how can I tackle this.
What can I do? Should I stick to my idea of decoupling getVideoMetadata() or should I nest it inside addFileToList() and wrestle with async/await?
It turns out that probing for metadata introduces a race condition, so we should structure the code so that the flow continues after the callback:
const ffmpeg = require('fluent-ffmpeg')
var data
ffmpeg.ffprobe(file.name, (err, metadata) => {
if (!err) {
data = metadata
continueDoingStuff()
}
})
I'm new to using csv-parse and this example from the project's github does what I need with one exception. Instead of outputting via console.log I want to store data in a variable. I've tried assigning the fs line to a variable and then returning data rather than logging it but that just returned a whole bunch of stuff I didn't understand. The end goal is to import a CSV file into SQLite.
var fs = require('fs');
var parse = require('..');
var parser = parse({delimiter: ';'}, function(err, data){
console.log(data);
});
fs.createReadStream(__dirname+'/fs_read.csv').pipe(parser);
Here is what I have tried:
const fs = require("fs");
const parse = require("./node_modules/csv-parse");
const sqlite3 = require("sqlite3");
// const db = new sqlite3.Database("testing.sqlite");
let parser = parse({delimiter: ","}, (err, data) => {
// console.log(data);
return data;
});
const output = fs.createReadStream(__dirname + "/users.csv").pipe(parser);
console.log(output);
I was also struggling to figure out how to get the data from csv-parse back to the top-level that invokes parsing. Specifically I was trying to get parser.info data at the end of processing to see if it was successful, but the solution for that can work to get the row data as well, if you need.
The key was to wrap all the stream event listeners into a Promise, and within the parser's callback resolve the Promise.
function startFileImport(myFile) {
// THIS IS THE WRAPPER YOU NEED
return new Promise((resolve, reject) => {
let readStream = fs.createReadStream(myFile);
let fileRows = [];
const parser = parse({
delimiter: ','
});
// Use the readable stream api
parser.on('readable', function () {
let record
while (record = parser.read()) {
if (record) { fileRows.push(record); }
}
});
// Catch any error
parser.on('error', function (err) {
console.error(err.message)
});
parser.on('end', function () {
const { lines } = parser.info;
// RESOLVE OUTPUT THAT YOU WANT AT PARENT-LEVEL
resolve({ status: 'Successfully processed lines: ', lines });
});
// This will wait until we know the readable stream is actually valid before piping
readStream.on('open', function () {
// This just pipes the read stream to the response object (which goes to the client)
readStream.pipe(parser);
});
// This catches any errors that happen while creating the readable stream (usually invalid names)
readStream.on('error', function (err) {
resolve({ status: null, error: 'readStream error' + err });
});
});
}
This is a question that suggests confusion about an asynchronous streaming API and seems to ask at least three things.
How do I get output to contain an array-of-arrays representing the parsed CSV data?
That output will never exist at the top-level, like you (and many other programmers) hope it would, because of how asynchronous APIs operate. All the data assembled neatly in one place can only exist in a callback function. The next best thing syntactically is const output = await somePromiseOfOutput() but that can only occur in an async function and only if we switch from streams to promises. That's all possible, and I mention it so you can check it out later on your own. I'll assume you want to stick with streams.
An array consisting of all the rows can only exist after reading the entire stream. That's why all the rows are only available in the author's "Stream API" example only in the .on('end', ...) callback. If you want to do anything with all the rows present at the same time, you'll need to do it in the end callback.
From https://csv.js.org/parse/api/ note that the author:
uses the on readable callback to push single records into a previously empty array defined externally named output.
uses the on error callback to report errors
uses the on end callback to compare all the accumulated records in output to the expected result
...
const output = []
...
parser.on('readable', function(){
let record
while (record = parser.read()) {
output.push(record)
}
})
// Catch any error
parser.on('error', function(err){
console.error(err.message)
})
// When we are done, test that the parsed output matched what expected
parser.on('end', function(){
assert.deepEqual(
output,
[
[ 'root','x','0','0','root','/root','/bin/bash' ],
[ 'someone','x','1022','1022','','/home/someone','/bin/bash' ]
]
)
})
As to the goal on interfacing with sqlite, this is essentially building a customized streaming endpoint.
In this use case, implement a customized writable stream that accepts the output of parser and sends rows to the database.
Then you simply chain pipe calls as
fs.createReadStream(__dirname+'/fs_read.csv')
.pipe(parser)
.pipe(your_writable_stream)
Beware: This code returns immediately. It does not wait for the operations to finish. It interacts with a hidden event loop internal to node.js. The event loop often confuses new developers who are arriving from another language, used to a more imperative style, and skipped this part of their node.js training.
Implementing such a customized writable stream can get complicated and is left as an exercise for the reader. It will be easiest if the parser emits a row, and then the writer can be written to handle single rows. Make sure you are able to notice errors somehow and throw appropriate exceptions, or you'll be cursed with incomplete results and no warning or reason why.
A hackish way to do it would have been to replace console.log(data) in let parser = ... with a customized function writeRowToSqlite(data) that you'll have to write anyway to implement a custom stream. Because of asynchronous API issues, using return data there does not do anything useful. It certainly, as you saw, fails to put the data into the output variable.
As to why output in your modified posting does not contain the data...
Unfortunately, as you discovered, this is usually wrong-headed:
const output = fs.createReadStream(__dirname + "/users.csv").pipe(parser);
console.log(output);
Here, the variable output will be a ReadableStream, which is not the same as the data contained in the readable stream. Put simply, it's like when you have a file in your filesystem, and you can obtain all kinds of system information about the file, but the content contained in the file is accessed through a different call.
Up until now for me the concept of a Node Worker has been one of those things that sounds interesting and I will find out about one day.
Well that day has come and I am asking for some help.
I need to call a web service and then process the data returned. I can call the service with an XMLHttpRequest, but then I have to get useful data out of it.
There is a nice node module that both calls the service and returns the data in a useful form with one call.
I can set up Node worker (in Wakanda) to do this and verify that it works.
My problem is handling the asynchronous call in the proxy.
The call to the node module looks like this:
myModule.getData(param, (err, data) => {
// data is an object containing everything I want.
// if I get data I want to return it to the proxy
// if I get an err I want to return the error to the proxy
});
So my wrapper code looks something like this:
function doSomething(param){
// call proxy with param
// wait for result
// return result
}
This all sounds like something I should know how to do. However I think I am struggling with too many new things and getting myself absolutely confused.
PS: I did try Threadify but couldn't work out how to get the worker to return the error it received.
I would really appreciate any help or pointers here.
If I am correctly understanding your issue, you cannot return a value from a function AFTER an asynchronous call completes. You need to handle the data within the myModule.getData callback.
If you would rather handle it in a calling function (like doSomething), you can use a promise to "return" a value.
function myProxy(param) {
return new Promise((resolve, reject) => {
myModule.getData(param, (err, data) => {
if (!err) { // or however you determine an error occurred.
resolve(data); // if I get data I want to return it to the proxy
} else {
reject(err); // if I get an err I want to return the error to the proxy
}
});
});
}
function doSomething(param){
myProxy(param).then(data => {
// Handle the data here.
}).catch(err => {
// Handle the error here.
});
}
I am trying to write a script that will query a local database and send the results to a Google Spreadsheet. Where I am currently stuck and looking for help is how do I get the results of my query.
var myQuery = [];
function runQuery(query, callback) {
client.connect(function (err) {
if (err) throw err;
client.query(query, function (err, result) {
if (err) {
console.log(err);
}
callback({ result: result });
client.end(function (err) {
if (err) throw err;
});
});
});
}
runQuery('SELECT item_number FROM item WHERE item_number LIKE \'11-%\' LIMIT 10', function (resultsObject) {
myQuery = JSON.stringify(resultsObject.result.rows);
console.log('Did stringify work?');
console.log(myQuery);
});
console.log(myQuery);
My output:
Info: Start process (3:46:11 PM)
[]
Did stringify work?
[{"item_number":"11-0011"},{"item_number":"11-0012"},{"item_number":"11-1255"},{"item_number":"11-0052"},{"item_number":"11-0060"},{"item_number":"11-1256"},{"item_number":"11-1281"},{"item_number":"11-0659"},{"item_number":"11-0660"},{"item_number":"11-0054"}]
Info: End process (3:46:12 PM)
I think I understand what's happening, the scope of the variable myQuery is set and printed within the runQuery function just fine, but outside of that function it's not being actually set. How do I get around this?
Very new to JavaScript and NodeJS so I hope I'm using the correct terminology.
Thanks in advance!
The empty myQuery has already been consoled before you actually put values into it. This issue is caused by Javascript's Asynchronous mechanism. If you want to do something with the result, you have to use a callback function. Just like what you did when you accessed result from runQuery.
Another way to do it is promise. You can return a promise in some function and use the promise to access result.
If you decide to use promise, here is the code:
function getresult(parameters) {
return new Promise((resolve, reject) => {
runQuery(parameter, resolve, reject);
});
}
Using this function, you passed resolve as callback to runQuery. You do want to add another err callback function to catch err in runQuery. In promise you use reject as err callback.
When you want to do something with the result:
getresult(parameters)
.then((result) => {put your code here. result variable is your result})
.catch((err) => {do whatever with err});
This is not a scope issue. It is a timing issue. The query takes some time to execute. JavaScript is 'non-blocking' meaning it will start an asynchronous process (like doing a DB query - really virtually all I/O operations) and return the result later. In your code you are using a callback that is run once the data becomes available.
So, with that in mind, the way to think about your code is that you are doing things in this order:
1) declaring your function runQuery - this happens first since in JavaScript function declarations are moved to the top of the script they are in (a process called hoisting)
2) your variable myQuery is established and initialized (to an empty array)
3) your query is kicked off by a call to runQuery; this only starts the process
4) your console.log(myQuery) at the end of your script is run
5) some time later the query completes and your callback is called with the data where your other console.log() shows the returned data
I have an event triggering a Metor.call():
Meteor.call("runCode", myCode, function(err, response) {
Session.set('code', response);
console.log(response);
});
But my runCode function inside the server's Metheor.methods has inside it a callback too and I can't find a way to make it return something to response in the above code.
runCode: function(myCode) {
var command = 'pwd';
child = exec(command, function(error, stdout, stderr) {
console.log(stdout.toString());
console.log(stderr.toString());
// I Want to return stdout.toString()
// returning here causes undefined because runCode doesn't actually return
});
// I can't really return here because I don't have yet the valuer of stdout.toString();
}
I'd like a way to have the exec callback return something as runCode without setInterval which would work, but as a hacky way in my opinion.
You should use Future from fibers.
See docs here : https://npmjs.org/package/fibers
Essentially, what you want to do is wait until some asynchronous code is run, then return the result of it in a procedural fashion, this is exactly what Future does.
You will find out more here : https://www.eventedmind.com/feed/Ww3rQrHJo8FLgK7FF
Finally, you might want to use the Async utilities provided by this package : https://github.com/arunoda/meteor-npm, it will make your like easier.
// load future from fibers
var Future=Npm.require("fibers/future");
// load exec
var exec=Npm.require("child_process").exec;
Meteor.methods({
runCode:function(myCode){
// this method call won't return immediately, it will wait for the
// asynchronous code to finish, so we call unblock to allow this client
// to queue other method calls (see Meteor docs)
this.unblock();
var future=new Future();
var command=myCode;
exec(command,function(error,stdout,stderr){
if(error){
console.log(error);
throw new Meteor.Error(500,command+" failed");
}
future.return(stdout.toString());
});
return future.wait();
}
});