I have some code in node.js using express. I route a call to request data from a mysql database, and what I want to do is pass that to another function to restructure the returned json from a tabular form (table query) to a hierarchy type json.
I've tested the script separately to restructure the output from my sql query. However, I am having trouble passing it from my query function to my new script(function)
I'm just not seeing what I am doing wrong. Any help please and thanks.
exports.get_site_menu = function (req, res) {
var dbc;
console.log('In menu setup');
async.waterfall([
// get a connection
function (callback) {
db.db(callback);
}
,
querylookup, modifyjson
], completed);
function querylookup(dbclient, res) {
dbc = dbclient;
dbc.query("SELECT categories, " +
"subcategories, " +
"pid, " +
"title, " +
"description " +
"FROM MENU_SELECT_ACTIVE_VIEW " +
"where company_id = ? and site_id = ?", [req.query.companyid, req.query.siteid], res);
}
function modifyjson(err, res) {
categories = [];
console.log('results ' + res);
res.forEach(function (entry) {
var cindex = categories.map(function (category) {
return category.name;
}).indexOf(entry.categories);
console.log(cindex);
if (cindex < 0) {
// Not found in categories array
cindex = categories.push({
name: entry.categories,
subcategories: []
}) - 1; // -1 to fix the index
}
// Lets search the subcategory
var category = categories[cindex];
var sindex = category.subcategories.map(
function (subcategory) {
return subcategory.name;
}
).indexOf(entry.subcategories);
if (sindex < 0) {
// Not Found
sindex = category.subcategories.push({
name: entry.subcategories,
items: []
}) - 1;
}
// Subcategory exists. Just push
category.subcategories[sindex].items.push({
pid: entry.pid,
description: entry.description,
title: entry.title
});
});
menu = {
menu: {
categories: categories
}
};
console.log('menu ' + menu);
}
function completed(err, menu, fields) {
if (dbc) dbc.end();
if (err) {
callback(err);
} else {
console.log(menu);
res.contentType('json');
res.send(JSON.stringify(menu));
}
}
};
You need to pass each result to own callback to pass next function. I have refactored your code like;
exports.get_site_menu = function (req, res) {
var dbc;
console.log('In menu setup');
async.waterfall([
// get a connection
function (callback) {
db.db(callback, some_result);
},
function querylookup(dbclient, res, callback) {
dbc = dbclient;
dbc.query("SELECT categories, " +
"subcategories, " +
"pid, " +
"title, " +
"description " +
"FROM MENU_SELECT_ACTIVE_VIEW " +
"where company_id = ? and site_id = ?", [req.query.companyid, req.query.siteid], res);
callback(null, result_from_queryLookup);
},
function modifyjson(err, res, callback) {
categories = [];
console.log('results ' + res);
res.forEach(function (entry) {
var cindex = categories.map(function (category) {
return category.name;
}).indexOf(entry.categories);
console.log(cindex);
if (cindex < 0) {
// Not found in categories array
cindex = categories.push({
name: entry.categories,
subcategories: []
}) - 1; // -1 to fix the index
}
// Lets search the subcategory
var category = categories[cindex];
var sindex = category.subcategories.map(
function (subcategory) {
return subcategory.name;
}
).indexOf(entry.subcategories);
if (sindex < 0) {
// Not Found
sindex = category.subcategories.push({
name: entry.subcategories,
items: []
}) - 1;
}
// Subcategory exists. Just push
category.subcategories[sindex].items.push({
pid: entry.pid,
description: entry.description,
title: entry.title
});
});
menu = {
menu: {
categories: categories
}
};
console.log('menu ' + menu);
callback(null, menu, fields);
}
], function completed(err, menu, fields) {
if (dbc) dbc.end();
if (err) {
callback(err);
} else {
console.log(menu);
res.contentType('json');
res.send(JSON.stringify(menu));
}
});
};
Especially, be careful on callback parts.
Related
I have pagination made with Sequelize
router.js
router.post('/getall', async (req, res) => {
try {
const { q, page, limit, order_by, order_direction } = req.query;
const { candidate, position, filters } = req.body
let include = [
{
model: SortedLvl,
where: {
lvl: filters.jobLvl.name,
months: { [Op.gte]: filters.jobMinExp.value },
},
},
{
model: SortedLastJob,
where: { jobposition: filters.jobType.name }
},
{
model: SortedSkills,
}
]
let search = {};
let order = [];
let filterCandidate = {}
if (candidate) {
if (candidate != undefined) {
t = candidate.split(/[ ,]+/)
let arr = new Array()
// console.log('t', t);
t.map((el, index) => {
console.log('el', el);
if (typeof el == 'number') {
arr.push({ first_name: { [Op.iLike]: `%` + `${el}` + `%` } }, { last_name: { [Op.iLike]: `%` + `${el}` + `%` } });
} else {
arr.push({ first_name: { [Op.iLike]: `%` + `%${el}%` + `%` } }, { last_name: { [Op.iLike]: `%` + `%${el}%` + `%` } });
}
});
filterCandidate = {
[Op.or]: arr
};
}
}
let filterPosition = {}
if (position) {
if (position != undefined) {
filterPosition = { position: { [Op.iLike]: `%${position}%` } }
}
}
if (filterCandidate.length > 0 || filterPosition.length > 0) {
search = { where: { ...(filterCandidate || []), ...(filterPosition || []) } }
}
console.log('search', search);
console.log('candidate', filterCandidate);
console.log('position', filterPosition);
if (order_by && order_direction) {
order.push([order_by, order_direction]);
}
const transform = (records) => {
return records.map(record => {
return {
id: record.id,
name: record.name,
date: moment(record.createdAt).format('D-M-Y H:mm A')
}
});
}
const products = await paginate(Candidate, page, limit, search, order, include);
return res.json({
success: true,
// message: '',
data: products
})
} catch (error) {
console.log('Failed to fetch products', error);
return res.status(500).send({
success: false,
message: 'Failed to fetch products'
})
}
});
paginate.js
const paginate = async (model, pageSize, pageLimit, search = {}, order = [], include, transform, attributes, settings) => {
try {
const limit = parseInt(pageLimit, 10) || 10;
const page = parseInt(pageSize, 10) || 1;
// create an options object
let options = {
offset: getOffset(page, limit),
limit: limit,
distinct: true,
include: include,
};
// check if the search object is empty
if (Object.keys(search).length) {
options = { ...options, ...search };
}
if (attributes && attributes.length) {
options['attributes'] = attributes;
}
if (order && order.length) {
options['order'] = order;
}
let data = await model.findAndCountAll(options);
if (transform && typeof transform === 'function') {
data = transform(data.rows);
}
return {
previousPage: getPreviousPage(page),
currentPage: page,
nextPage: getNextPage(page, limit, count),
total: count,
limit: limit,
data: data.rows
}
} catch (error) {
console.log(error);
}
}
const getOffset = (page, limit) => {
return (page * limit) - limit;
}
const getNextPage = (page, limit, total) => {
if ((total / limit) > page) {
return page + 1;
}
return null
}
const getPreviousPage = (page) => {
if (page <= 1) {
return null
}
return page - 1;
}
module.exports = paginate;
First problem - on queries with included models sometimes it response with wrong count
and also it is so slow
Please, help me with solving this issue
Thank you
I tried use separate: true, required: true and etc - I just get empty arrays on included
In your sequelize request there are lots of json query. Mysql cant create directly index your json fields. You can create a referance your query property like below and after that you can add a index for reference column.
And change your referance column as created reference. Also you can create mysql index another requests columns.
ALTER TABLE [table_name] ADD COLUMN email VARCHAR(255)
GENERATED ALWAYS as (properties->>"$.[jsonObjectName].[jsonProperty]");
ALTER TABLE [table_name] ADD INDEX email ([jsonProperty]) USING BTREE;
I am trying to check whether a table entry exists in a database, but what I have so far always returns true even if there is no entry. What am I doing wrong?
Thank you all.
This will always console.log >>>> true
let myPersLocalEntityExistsPromise = function (localEntityGUID) {
return new Promise(function (resolve, reject) {
let myEntityExists = true;
const client = myDb.mySQLDbLocalConnection();
let stmt = "SELECT EXISTS(SELECT 1 FROM MY_ENTITY_TABLE WHERE LOCAL_MY_ENTITY_GUID = ?)";
let todo = [
localEntityGUID
];
client.query(stmt, todo, function (err, row) {
if (err) {
return ("myPersLocalEntityExists: " + err.message);
} else {
if (row && row.length) {
console.log(localEntityGUID + ' Case row was found!');
} else {
myEntityExists = false;
console.log(localEntityGUID + ' Case row was NOT found!');
}
}
});
client.end(function (err) {
if (err) {
console.log('ERRR');
}
});
if (myEntityExists) {
resolve(myEntityExists);
} else {
reject(myEntityExists);
}
})
}
function myPersLocalEntityExists(localEntityGUID, callback) {
let responsePromises = []
responsePromises.push(myPersLocalEntityExistsPromise(localEntityGUID))
Promise.all(responsePromises).then(fromResolve => {
console.log(">>>> " + fromResolve);
return fromResolve;
}).catch(fromReject => {
console.log(">>>> " + fromReject);
return fromReject;
});
}
I still do not handle well the asynchronous functions, I have an array of items, and I'm trying to for each item calculate some values and push to another array outside of the async function. Then I want to make some statistics calculation and send to front end. It's server side, nodejs handler, my code:
exports.register = function (plugin, options, next) {
function isInArray(value, array) {
return array.indexOf(value) > -1;
}
function statistics(values) {
var sum = math.sum(values);
var max = math.max(values);
var min = math.min(values);
var stddev = math.std(values);
var mean = math.mean(values);
var count = values.length;
}
plugin.route({
method: 'GET',
path: '/statistics/{orgId}/layout/{layoutId}',
config: {
pre: [
authorize(hasRole(['OPERATIONAL', 'STRATEGIC', 'LOP', 'TACTICAL']))
],
handler: function (request, reply) {
Category.find()
.where('organization')
.equals(request.params.orgId)
.exec(function (err, categories) {
var weight = [];
var price = [];
var volume = [];
var thisAR = [];
if (err || categories === null) {
return reply(Boom.badRequest('Categoria inexistente'));
} else {
Location.findById(request.params.layoutId)
.exec(function (err, layout) {
if(err) {
console.log(err);
}
var searchItems = function searchItems(category, next) {
Item.find()
.where('category')
.equals(category._id)
.exec(function (err, items) {
if (err) {
console.log(err);
} else {
var valuesToCalculate = [];
var itemsFiltered = [];
_.forEach(items, function(item) {
if(item.location && item.location !== null) {
if(isInArray(item.location.toString(), layout.contents)) {
itemsFiltered.push(item);
}
}
});
valuesToCalculate.push(itemsFiltered.length * category.data.weight);
valuesToCalculate.push(itemsFiltered.length * category.data.price);
valuesToCalculate.push(itemsFiltered.length * category.data.volume);
next(valuesToCalculate);
}
});
}
var onFinish = function onFinish(value, err) {
if(err) {
console.log(err);
}
console.log(value);
thisAR.push.apply(value);
}
async.each(categories, searchItems, onFinish);
console.log(thisAR);
//var arrays = [statistics(weight), statistics(price), statistics(volume)];
//return arrays;
});
}
});
}
}
});
next();
};
A few things stand out to me about this. First, you only call reply if there is an err or categories is null. Also, you are attempting to pass a non null value to an async.each callback. According to this: https://github.com/caolan/async#eacharr-iterator-callback, "if no error has occurred, the callback should be run without arguments or with an explicit null argument". I think you may misunderstand how the onFinish callback works with async.each.. it is not called for each item, it is called when all of the iterator functions have completed. So, rather than pushing items onto thisAR in onFinish, you should do so inside searchItems. I think this should work:
exports.register = function (plugin, options, next) {
function isInArray(value, array) {
return array.indexOf(value) > -1;
}
function statistics(values) {
var sum = math.sum(values);
var max = math.max(values);
var min = math.min(values);
var stddev = math.std(values);
var mean = math.mean(values);
var count = values.length;
}
plugin.route({
method: 'GET',
path: '/statistics/{orgId}/layout/{layoutId}',
config: {
pre: [
authorize(hasRole(['OPERATIONAL', 'STRATEGIC', 'LOP', 'TACTICAL']))
],
handler: function (request, reply) {
Category.find()
.where('organization')
.equals(request.params.orgId)
.exec(function (err, categories) {
var weight = [];
var price = [];
var volume = [];
var thisAR = [];
if (err || categories === null) {
return reply(Boom.badRequest('Categoria inexistente'));
} else {
Location.findById(request.params.layoutId)
.exec(function (err, layout) {
if(err) {
console.log(err);
}
var searchItems = function searchItems(category, next) {
Item.find()
.where('category')
.equals(category._id)
.exec(function (err, items) {
if (err) {
console.log(err);
} else {
var valuesToCalculate = [];
var itemsFiltered = [];
_.forEach(items, function(item) {
if(item.location && item.location !== null) {
if(isInArray(item.location.toString(), layout.contents)) {
itemsFiltered.push(item);
}
}
});
valuesToCalculate.push(itemsFiltered.length * category.data.weight);
valuesToCalculate.push(itemsFiltered.length * category.data.price);
valuesToCalculate.push(itemsFiltered.length * category.data.volume);
thisAR.push.apply(valuesToCalculate);
}
next(err);
});
}
var onFinish = function onFinish(err) {
if(err) {
console.log(err);
}
console.log(thisAR);
// call reply here
}
async.each(categories, searchItems, onFinish);
console.log(thisAR);
//var arrays = [statistics(weight), statistics(price), statistics(volume)];
//return arrays;
});
}
});
}
}
});
next();
};
I can't comment since I'm new but it seems that just removing the .apply in thisAR.push.apply(valuesToCalculate); will return a full array to your console.log.
I am angularjs newbie. I try to use ionic framework to do a practice, so that will use angularjs. And I got a little problem. I want to before insert data to check this data have exist, if not exist that will insert a new data.
On this method getContent.then(function(res){}), I will check this return res.length, if equal 0 I want insert this data. When I run, that will execute this console, and obj is have data. But at finally, I want get all data, but the data is empty.
But I found If I remove this insert method outside getContent.then(function(res){}), it's work. I have no idea how to fix this problem and cause this reason.
Thanks your help.
This is my Controller code
angular.module('starter.controllers', ['sqlite.services'])
.controller('TestCtrl', function($scope, $http, Tests, SQLService) {
SQLService.setup();
var new_tests = new Array();
$http.get('https://XXXXX').then(function(response){
var datas = response.data;
for (data_ in datas) {
var obj = {
id: datas[data_].content_id,
title: datas[data_].title,
url: datas[data_].url,
content: datas[data_].content
};
var getContent = SQLService.get_one(obj.id);
getContent.then(function(res) {
console.log('res ' , res);
console.log('res ' , res.length); // length get 0
if(res.length == 0) {
console.log('insert obj ' , obj);
console.log('SQLService ' , SQLService);
SQLService.insert(obj);
}
else if (res.length == 1) {
console.log('edit obj ' , obj);
}
});
// SQLService.insert(obj); // If I write insert code here is work for me
new_tests.push(obj);
}
})
.finally(function() {
SQLService.all().then(function (results) {
$scope.tests = results;
console.log('results ' , results);
});
});
This is my sql_service.js
angular.module('sqlite.services', [])
.factory('SQLService', function($q) {
var db;
function createDB() {
try {
if (window.cordova) {
$cordovaSQLite.deleteDB("my.db");
db = $cordovaSQLite.openDB({name: 'my.db'}); // device
}
else{
db = window.openDatabase("my.db", '1', 'my', 1024 * 1024 * 100); // browser
}
db.transaction(function(tx) {
tx.executeSql("CREATE TABLE IF NOT EXISTS pixnet (id integer not null primary key autoincrement, content_id text, title text, url, text, content text)", []);
});
}
catch(err) {
console.log('Error processing SQL: ' + err);
}
console.log('database created');
}
function insertNewContent(newContent) {
console.log('--insert--');
return promisedQuery("INSERT INTO pixnet (content_id, title, url, content) VALUES ('" + newContent.id + "', '" + newContent.title + "', '" + newContent.url + "', '" + newContent.content + "')", defaultResultHandler, defaultErrorHandler);
}
function getContents() {
return promisedQuery("SELECT * FROM pixnet", defaultResultHandler, defaultErrorHandler);
}
function updateContent(content){
console.log('update content ' , content);
return promisedQuery("UPDATE pixnet SET title='" + content.title + "', content='" + content.content + "' WHERE content_id = '" + content.id + "'", defaultResultHandler, defaultErrorHandler);
}
function getContent(content_id) {
return promisedQuery("SELECT * FROM pixnet WHERE content_id = '" + content_id + "'", defaultResultHandler, defaultErrorHandler);
}
function defaultResultHandler(deferred) {
return function(tx, results) {
console.log('defaultResultHandler results ' , results);
var len = results.rows.length;
var output_results = [];
for (var i=0; i<len; i++){
var t = {
'id': results.rows.item(i).id,
'content_id': results.rows.item(i).content_id,
'title': results.rows.item(i).title,
'url': results.rows.item(i).url,
'content': results.rows.item(i).content
};
output_results.push(t);
}
deferred.resolve(output_results);
}
}
function defaultErrorHandler(deferred) {
return function(tx, results) {
var len = 0;
var output_results = '';
deferred.resolve(output_results);
}
}
function promisedQuery(query, successCB, errorCB) {
var deferred = $q.defer();
db.transaction(function(tx){
tx.executeSql(query, [], successCB(deferred), errorCB(deferred));
}, errorCB);
return deferred.promise;
}
return {
setup: function() {
return createDB();
},
insert: function(content) {
return insertNewContent(content);
},
edit: function(content) {
return updateContent(content);
},
get_one: function(content_id) {
return getContent(content_id);
},
all: function() {
return getContents();
}
}
});
I believe what's happening is that the 'deferred' you create in promiseQuery is never resolved:
function promisedQuery(query, successCB, errorCB) {
var deferred = $q.defer();
db.transaction(function(tx){
tx.executeSql(query, [], successCB(deferred), errorCB(deferred));
}, errorCB);
return deferred.promise;
}
Since you are using cordova sqlite plugin, looking at the source code we see the third argument of the 'transaction' function is the success callback.
https://github.com/brodysoft/Cordova-SQLitePlugin/blob/master/www/SQLitePlugin.js#L74
So this means you want to resolve your promise in either of those callbacks. Try the following:
function promisedQuery(query, successCB, errorCB) {
var deferred = $q.defer();
db.transaction(function(tx){
tx.executeSql(query, [], successCB(deferred), errorCB(deferred));
}, errorCB, deferred.resolve);
return deferred.promise;
}
Passing the deferred.resolve function into the success callback (the last argument of transaction) will get it called when the transaction finishes.
angular.module('starter.controllers', ['sqlite.services'])
.controller('TestCtrl', function($scope, $http, Tests, SQLService) {
SQLService.setup();
var new_tests = new Array();
var call_async_in_loop = function(obj) {
var getContent = SQLService.get_one(obj.id);
getContent.then(function(res) {
console.log('res ', res);
console.log('res ', res.length); // length get 0
if (res.length == 0) {
console.log('insert obj ', obj);
console.log('SQLService ', SQLService);
SQLService.insert(obj);
} else if (res.length == 1) {
console.log('edit obj ', obj);
}
});
}
$http.get('https://XXXXX').then(function(response) {
var datas = response.data;
for (data_ in datas) {
var obj = {
id: datas[data_].content_id,
title: datas[data_].title,
url: datas[data_].url,
content: datas[data_].content
};
call_async_in_loop(obj)
new_tests.push(obj);
}
})
.finally(function() {
SQLService.all().then(function(results) {
$scope.tests = results;
console.log('results ', results);
});
});
You loosing the reference to obj because of the async call SQLService.get_one(obj.id). When the promise is resolved the for loop is already finished. So u have to create a closure to keep reference to obj.
I am using the flickrapi for NodeJS and keep running into the error listed in the title.
Everything works fine on my local machine (OSX 10.10.1), but Azure chokes every time. The app runs, but anything that uses the flickrapi chokes. Example below:
var express = require('express');
var router = express.Router();
var Flickr = require("flickrapi"),
flickrOptions = {
api_key: '**********',
secret: '***********'
};
router.get('/galleries', function(req, res) {
Flickr.tokenOnly(flickrOptions, function(error, flickr) {
var albums = [];
flickr.photosets.getList({ user_id: '*********' }, function(err, result) {
result.photosets.photoset.forEach(function(set){
albums.push(new Album(set));
});
albums.forEach(function(album, index, array) {
flickr.photosets.getPhotos({ photoset_id: album.id }, function(err, pResult) {
addPhotosToAlbum(pResult.photoset.photo, album, function() {
if (index === array.length - 1) {
res.json({ albums: albums });
}
});
});
})
});
});
});
function Album(photoset) {
this.id = photoset.id;
this.title = photoset.title._content;
this.photos = [];
}
function Photo(photo) {
this.url = new URL(photo);
}
function URL(photo) {
var url = 'http://farm' + photo.farm + '.static.flickr.com/' + photo.server + '/';
url += photo.id + '_' + photo.secret + '_';
this.thumbnail = url + 't.jpg';
this.small = url + 'm.jpg';
this.medium = url + 'z.jpg';
this.large = url + 'k.jpg';
}
function addPhotosToAlbum(photos, album, finish) {
photos.forEach(function(photo, index, array) {
album.photos.push(new Photo(photo));
if (index === array.length - 1) {
finish();
}
});
}
module.exports = router;
Update 1
I removed everything but the initial query, even that throws an error:
router.get('/galleries', function(req, res) {
Flickr.tokenOnly(flickrOptions, function(error, flickr) {
flickr.photosets.getList({ user_id: '*********' }, function(err, result) {
res.json({ albums: result });
});
});
});