I have this piece of code
module.exports = function(file, callback) {
var output, pdf, result, stderr;
output = '';
stderr = '';
result = [];
pdf = child_process.spawn('pdftotext', ['-layout', '-enc', 'UTF-8', file.name, '-']);
pdf.stdout.setEncoding('utf8');
pdf.stderr.setEncoding('utf8');
pdf.stdout.on('data', function(data) {
if (data) {
output += data;
}
});
pdf.stderr.on('data', function(data) {
if (data) {
stderr += data;
}
});
return pdf.on('close', function(code) {
var last_page, pages;
if (code !== 0) {
return stderr;
} else {
pages = output.split(/\f/);
if (!pages) {
return 'Nenhum texto foi encontrado';
}
last_page = pages[pages.length - 1];
if (!last_page) {
pages.pop();
}
file.text += pages;
return callback(file);
}
});
};
Basically I'm taking the text of PDFs using the pdftotext bash program.
This function is being called inside a Promise.map, so I need to return a new array of files, but before return, I need to call the callback, each will be responsible for other things.
The problem is: how can I return the pdf.on('close', funct....) ??
Inside it, I have the return callback(file); each is returning exactly what I want, but now I need return the same thing from pdf.on....
Thanks.
Your function, being asynchronous, should return a promise. Scrap that callback.
module.exports = function(filename) {
var output = '';
var stderr = '';
var pdf = child_process.spawn('pdftotext', ['-layout', '-enc', 'UTF-8', filename, '-']);
pdf.stdout.setEncoding('utf8');
pdf.stderr.setEncoding('utf8');
pdf.stdout.on('data', function(data) {
if (data) output += data;
});
pdf.stderr.on('data', function(data) {
if (data) stderr += data;
});
return new Promise(function(resolve, reject) {
//^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pdf.on('close', function(code) {
if (code)
reject(new Error(stderr, code)); // call reject() for errors…
else
resolve(output); // and resolve() with the result *instead of that `callback`*
});
}).then(function(result) { // so that you then can chain processing to
var pages = result.split(/\f/);
if (!pages.length) {
throw new Error('Nenhum texto foi encontrado');
}
var last_page = pages[pages.length - 1];
if (!last_page)
pages.pop();
return pages.join("");
});
};
With that, you can easily use it as a proper Promise.map callback:
function phase_one(files) {
return Promise.map(files, function(file) {
return pdf(file.name).then(function(text) {
file.text = text;
return file;
});
}, {
concurrency: 3000
});
}
Related
I am trying to read some data from 2 different tables and parse a CSV file before rendering an ejs file.
I can get the data from both tables and from the CSV file but I seem to be unable to return the result.
Pretty sure this is a problem with the way I handle async execution but I fail to see what I am doing wrong.
I've spent the last 2 days reading about this (including the threads around here) and browsing but somehow the answer still escapes me.
First file - usercms.js
app.get('/userscms', function(req, res)
{
existingUsers.getExistingUsers()
.then(function(appUsers)
{
//global users array
//I can display these in my ejs file
globalAppUsers = appUsers;
})
.then(existingUsersAttributesQlik.getExistingUsersAttributesQlik())
.then(function(usersQlikAttributes)
{
//global user attributes array
//undefined data
globalUsersQlikAttributes = usersQlikAttributes;
})
.then(existingSuppliers.parseSuppliersCSV())
.then(function(supplierData)
{
//the result I am expecting
//this prints undefined
console.log(supplierData);
}).then(function()
{
res.render('userscms.ejs',
{
users: globalAppUsers,
attributes: globalUsersQlikAttributes
});
});
});
Second function - getxistingUsers.js (identical to the getExistingUsersAttributesQlik, except for the query)
var userData = [];
var appUsers = [];
(function (exports)
{
exports.getExistingUsers = function ()
{
return promisemysql.createConnection(dbconfig.development).then(function(conn)
{
var result = conn.query("SELECT id, username, firstName, lastName, email, phone, lastLogin, isAdmin, isValid, isPhoneValid, accountCreationDateTime FROM Users");
conn.end();
return result;
}).then(function(rows)
{
return rows;
}).then(function(rows)
{
if (rows.length)
{
userData = [];
appUsers = [];
rows.forEach(function (elem)
{
userData.push(_.toArray(elem));
});
for (i = 0; i < userData.length; i++)
{
var appUser = new appUserModel.AppUser(
userData[i][0],
userData[i][1],
userData[i][2],
userData[i][3],
userData[i][4],
userData[i][5],
userData[i][6],
userData[i][7],
userData[i][8],
userData[i][9],
userData[i][10]);
appUsers.push(_.toArray(appUser));
}
return appUsers;
}
else
{
console.log("NOPE");
return null;
}
}).then(function(appUsers)
{
console.log(appUsers);
return appUsers;
});
};
})(typeof exports === 'undefined' ? this['getExistingUsers'] = {} : exports);
Third file - parseSuppliersCSV.js
var supplierData = [];
var suppliersData = [];
var csvCount = 0;
(function (exports)
{
exports.parseSuppliersCSV = function ()
{
return new Promise(function(resolve, reject)
{
var fileStream = fs.createReadStream("myCSV.csv");
var parser = fastCsv();
csvCount = 0;
supplierData = [];
suppliersData = [];
fileStream
.on("readable", function ()
{
var data;
while ((data = fileStream.read()) !== null)
{
parser.write(data);
}
})
.on("end", function ()
{
parser.end();
});
parser
.on("readable", function ()
{
var data;
while ((data = parser.read()) !== null)
{
if(csvCount >= 1)
{
csvCount++;
var arrayOfStrings = data[0].split(';');
var supplier = new supplierModel.Supplier(arrayOfStrings[0],arrayOfStrings[1]);
suppliersData.push(_.toArray(supplier));
}
else
{
csvCount++;
}
}
})
.on("end", function ()
{
console.log("done");
//all OK here
console.log(suppliersData);
//this doesn't seem to return anything
return suppliersData;
});
});
};
})(typeof exports === 'undefined' ? this['parseSuppliersCSV'] = {} : exports);
Any ideas what I am doing wrong? Am I approaching this the wrong way?
I'll take a guess here and assume the promise you created should resolve to something...instead of returning a value.
.on("end", function ()
{
console.log("done");
//all OK here
console.log(suppliersData);
//this doesn't seem to return anything
return resolve(suppliersData);
});
I am trying to iterate through the JSON files generated by the protractor tests. I pull all the file names into an array and call a method that opens and parses through the each file, post the results to the database and pass back a passed/failed flag.
I have tried all the examples here
Make angular.forEach wait for promise after going to next object and still get the same results.
The method is actually called, but the results are not posted to the db. I have tested the parser.parseResults on an individual file and it successfully posted to the db, so it has to have something to do with the promise not resolving correctly.
Is it not possible to do something like this in the jasmine/protractor framework? Or do I have something wrong in the code?
I have included the code for my latest attempt.
Thank You
Christine
matches.reduce(function (p, val) {
console.log('val', val);
return p.then(function () {
return parser.parseResults(val);
});
}, Promise.resolve()).then(function (finalResult) {
console.log('finalResult = ', finalResult);
}, function (err) {
console.log('error in reduce',err);
});
parser.parseResults code
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
try {
if (err != null) {
console.log('error reading file',err);
reject(err);
}
console.log('obj - ',obj);
var results = [];
var Passed = 0;
var Message = '';
var Stack = '';
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
if (specs.length > 0) {
for (var i = 0; i < specs.length; i++) {
var assert = specs[i];
var tcR = new RegExp(/TC[\d]+/);
var tc = assert.description.match(tcR);
if (!assert.failedExpectations.length) {
Passed = 1;
}
else {
assert.failedExpectations.forEach((expectation) => {
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
})
Passed = 0;
}
if (tc != null) {
utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild,
'P', Message, Stack, 0, moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss'), '')
.then(function (resp) {
resolve(Passed);
}, (err) => {
console.log('Posting to Database failed ', err);
reject(err);
});
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
reject(err);
}
}
}
}
}
catch (err) {
console.log('rejecting opening file');
reject(err);
}
});
})
}
If there is not exactly one suite in the obj, with exactly one spec, then your promise is either resolved not at all or multiple times.
Avoid wrapping too many things in the new Promise constructor - always promisify on the smallest possible level, and use promise chaining afterwards.
protractorParser.prototype.parseResults = function (fileName) {
return new Promise((resolve, reject) => {
console.log('In parseresults', fileName);
json.readFile(fileName, function (err, obj) {
if (err != null) {
console.log('error reading file', err);
reject(err);
} else {
resolve(obj);
}
});
}).then(function(obj) {
console.log('obj - ',obj);
var results = [];
for (var suite in obj) {
var specs = obj[suite].specs;
console.log('spec - ', specs);
for (let i = 0; i < specs.length; i++) {
const assert = specs[i];
const tcR = /TC[\d]+/;
const tc = assert.description.match(tcR);
let Passed = 1;
let Message = '';
let Stack = '';
if (assert.failedExpectations.length) {
const expectation = assert.failedExpectations[assert.failedExpectations.length-1];
Passed = 0;
Message = expectation.message;
Stack = expectation.stack.split('\n')[1].trim();
}
if (tc != null) {
const time = moment().utcOffset(config.get('settings.timeOffset')).format('YYYY-MM-DDTHH:mm:ss');
const promise = utility.TestDataManager.insertAutomationResults(tc[0], assert.description, Passed, process.env.testBuild, 'P', Message, Stack, 0, time, '');
results.push(promise.catch(err => {
console.log('Posting to Database failed ', err);
throw err;
}));
} else {
console.log('no test case found for test: ' + assert.description + ' -- skipping');
// I don't think you want to `throw err` here, right?
}
}
}
return Promise.all(results);
});
};
How can I wait for the Braintree Transaction.search() function to return all data.
Right now it does not wait and just comes back with undefined return value.
Here is the code
I tried to use Meteor.asynwrap but that also does not work.
`
function getTrxns(cid) {
var future = new Future();
var trxns = [];
var i = 0
var stream = gateway.transaction.search(function (search) {
r = search.customerId().is(cid)});
stream.on("data", function(data){
i = i+1
trxns.push({
'id':data.id,
'amount':data.amount,
'crtDt': data.createdAt,
'ccType': data.creditCard.cardType,
'currency': data.currencyIsoCode,
'last4': data.creditCard.last4,
'expdt': data.creditCard.expirationDate
});
});
stream.on("end", function(){
// print the output in console
console.log('End Stream cnt: '+i);
return trxns;
});
stream.resume();
}
Meteor.methods({
findCustTrxns: function() {
var btId = Meteor.user().custBtId;
if (!btId) { return []; };
console.log('findCustTrxns cusBtId: '+btId);
var xx = getTrxns(btId);
console.log('xx len :'+xx.length);
}
});
OUTPUT is:
I20170509-15:22:09.095(0)? findCustTrxns cusBtId: 232057823
I20170509-15:22:09.095(0)? Exception while invoking method 'findCustTrxns' TypeError: Cannot read property 'length' of undefined
I20170509-15:22:09.095(0)? End Stream cnt: 56
Found a way to make it work:
1. Added a callback function
function getTrxns(cid,callback )
2. invoked the callback in stream.on('end;..) Here is the code
function getTrxns(cid,callback ) {
var trxns = [];
var i = 0
var stream = gateway.transaction.search(function (search) {
r = search.customerId().is(cid)});
stream.on("data", function(data){
i = i+1
trxns.push({
'id':data.id,
});
});
stream.on("end", function(){
// print the output in console
console.log('End Stream cnt: '+i);
callback('', trxns);
});
stream.resume();
}
3. Changed the Meteor Method :
findCustTrxns: function(btId) {
if (!btId) { return []; };
console.log('findCustTrxns cusBtId: '+btId);
var trxns = [];
var i = 0;
var fn = Meteor.wrapAsync(getTrxns); //made it a synchronous call
try {
var res = fn(btId);
if (res) {
console.log('Got data from getTrxns ');
return res;
}
} catch( err) {
console.log('Error calling hetTrxns '+err);
}
}, //findCustTrxns
Now I am able to get the Transactions. Hope it helps
I suspect this is very trivial but I have this code:
var latestMth;
d3.csv("data/blah.csv", function(rows) {
x = rows.map(function(d) {
return parseDate(d.dt)
});
latestMth = formatDateOutputMTH(d3.max(d3.values(x)));
});
...
...
.text("Month is " + latestMth);
It works ok displaying "Month is Mar17".
I've attempted to make the above a function but the result now comes through as "Month is undefined". Why is this happening:
var latestMth = function() {
d3.csv("data/blah.csv", function(rows) {
x = rows.map(function(d) {
return parseDate(d.dt)
});
return formatDateOutputMTH(d3.max(d3.values(x)));
});
}
...
...
.text("Month is " + latestMth());
Assuming d3.csv() is not async.
Because you don't return from your function:
var latestMth = function() {
return d3.csv("data/blah.csv", function(rows) { // added return
x = rows.map(function(d) {
return parseDate(d.dt)
});
return formatDateOutputMTH(d3.max(d3.values(x)));
});
}
Note, if it's async, you can use a Promise:
function setLatestMonth() {
return new Promise((resolve, reject) => {
d3.csv("data/blah.csv", function(rows) {
x = rows.map(function(d) {
return parseDate(d.dt)
});
resolve(formatDateOutputMTH(d3.max(d3.values(x))));
});
});
}
setLatestMonth().then(latestMonth => {
// ...
// ...
// ...
.text("Month is " + latestMonth);
});
I don't know d3.csv() - if it already returns a promise, you can simply chain that one instead of creating a new Promise by yourself.
How to pass data from promise then method to an object method.
this.httpReq(url).then(function (data) {
this.storeData(data);
});
I know that here I'm out of scope and this doesn't refer to my object. But, nevertheless, I can't understand how to resolve it.
Bellow you can find entire code snippet. In the end, I want to get data from the service API and store it in the object array property this.storage.
var http = require('http');
function CoubApi (url) {
this.url = url;
this.storage = [];
this.httpReq = httpReq;
this.searchData = searchData;
this.storeData = storeData;
}
function httpReq (url) {
var promise = new Promise (function (resolve, reject) {
http.get(url, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
if(data.length > 0) {
resolve(JSON.parse(data));
} else {
reject("Error: HTTP request rejected!");
}
});
}).on('error', function (err) {
console.log("Error: ", e);
});
});
return promise;
}
function storeData (data) {
var i;
console.log("Storrrreee");
for(i = 0; i < 10; i++) {
this.storage.push(data.coubs[i]);
}
}
function searchData (searchtext, order, page) {
var url = this.url+
"search?q="+searchtext+
"&order_by="+order+
"&page="+page;
this.httpReq(url).then(function (data) {
this.storeData(data);
});
}
var coub = new CoubApi("http://coub.com/api/v2/");
coub.searchData("cat", "newest_popular", 1);
console.log(coub.storage);
You can store this in varaible:
var self = this;
this.httpReq(url).then(function (data) {
self.storeData(data);
});
or use bind:
this.httpReq(url).then(function (data) {
this.storeData(data);
}.bind(this));