We Keep Coding

How to use foreign keys in a 1:M relationship when implementing with .bulkCreate() using SequelizeJS and PostgreSQL

I'm writing a script that can extract data from (.csv) datasets. I am able to pull the data into the console, create new tables and insert the data into the DB.
I want to figure out how to set the foreign key while implementing .bulkCreate(). When I stick to regular .create() I get a resource error.
Datasets are held inside 2 arrays:
competitorObjs
meetObjs
So far this is how I'm trying to insert the data:
sequelize.sync().then(() => {
meets.bulkCreate(meetObjs).then(data => {
competitorObjs.forEach(x => {
competitors.create({
MeetID: x.MeetID,
Name: x.Name,
Sex: x.Sex,
Equipment: x.Equipment,
Age: x.Age,
Division: x.Division,
BodyweightKg: x.BodyweightKg,
WeightClassKg: x.WeightClassKg,
BestSquatKg: x.BestSquatKg,
BestBenchKg: x.BestBenchKg,
BestDeadlift: x.BestDeadlift,
TotalKg: x.TotalKg,
Place: x.Place,
Wilks: x.Wilks,
UserId: data.get("MeetID") // Set FK here (Not sure if correct implementation)
})
})
}).then(() => {
console.log("Bulk Creation Success!");
}).catch((err) => {
console.log(err)});
})
When the script is finished Only the 'Meets' table has populated but 'Competitors' stay empty.
How do I set Foreign Keys in each 'Competitor' insertion to point to each Primary Key inside the 'Meets' table? (FKs are not set to unique)

You are almost there. You can loop with an index
competitorObjs.forEach((x, index) => {
competitors.create({
...
..
..
UserId: data[index]["MeetID"]
The problem with this approach is that for creating N objects, you would create N + 1 queries. 1 for the bulk create and N for associated table. You can use bulkCreate in the associated array as well.
You can iterate over the competitorObjs, fill in the UserId as mentioned above and then do bulkCreate.

Posting the solution or at least what let me populate the competitors table.
For some reason I thought we set the foreign key during the insertion.
At the top, I think I set the FK inside the 'competitors' model as 'MeetID'
meets.hasMany(competitors, {
foreignKey: "MeetID",
constraints: false
});
var meetObjs = [];
var competitorObjs = [];
// Extract meets data to meetObjs
// confirm connection...
sequelize.authenticate().then(() => {
console.log("Data base connection established")
}).then(() => {
// read contents of file...
fs.readFile(path.join(__dirname, "/data/meets.csv"), "utf-8", (err, data) => {
if (err) {
console.log(err);
} else {
// begin parsing data...
Papa.parse(data, {
delimiter: ',',
newline: '\n',
complete: function (results) {
// push to meetObjs array
for (let i = 1; i < results.data.length - 1; i++) { //NOTE: Last element extracted is NaN. Look more into this.
meetObjs.push({
MeetID: results.data[i][0],
MeetPath: results.data[i][1],
Federation: results.data[i][2],
Date: results.data[i][3],
MeetCountry: results.data[i][4],
MeetState: results.data[i][5],
MeetTown: results.data[i][6],
MeetName: results.data[i][7]
})
}
// read contents of file...
fs.readFile(path.join(__dirname, "/data/openpowerlifting.csv"), "utf-8", (err, data) => {
if (err) {
console.log(err);
} else {
// begin parsing data...
Papa.parse(data, {
delimiter: ',',
newline: '\n',
complete: function (results) {
// push to meetObjs array
for (let i = 1; i < results.data.length - 1; i++) { //NOTE: Last element extracted is NaN. Look more into this.
competitorObjs.push({
MeetID: Number(results.data[i][0]),
Name: results.data[i][1],
Sex: results.data[i][2],
Equipment: results.data[i][3],
Age: Number(results.data[i][4]),
Division: results.data[i][5],
BodyweightKg: Number(results.data[i][6]),
WeightClassKg: results.data[i][7],
Squat4Kg: Number(results.data[i][8]),
BestSquatKg: Number(results.data[i][9]),
Bench4Kg: Number(results.data[i][10]),
BestBenchKg: Number(results.data[i][11]),
Deadlift4Kg: Number(results.data[i][12]),
BestDeadlift: Number(results.data[i][13]),
TotalKg: Number(results.data[i][14]),
Place: results.data[i][15],
Wilks: Number(results.data[i][16])
})
}
sequelize.sync().then(() => {
meets.bulkCreate(meetObjs).then(() => {
competitors.bulkCreate(competitorObjs).then(() => {
console.log("Competitors created!");
}).catch("Competitors creation failed!")
}).then(() => {
console.log("Meets created!");
}).catch(() => {
console.log("Meets creation failed!");
})
})
}
});
}
});
}
});
}
});
}).catch(() => {
console.log("Something went wrong trying to connect to the DB");
});

Node.js code only working at top of file with sequelize

When at the top of my server-side code, this works fine and the results produced are correct:
var data_playlists = {};
models.Playlist.findAll({
attributes: ['id', 'name']
}).then(function (playlists){
data_playlists['playlists'] = playlists.map(function(playlist){
return playlist.get({plain: true})
});
addsongs(data_playlists, 1);
addsongs(data_playlists, 2);
addsongs(data_playlists, 3);
});
but when it's inside one of my Express methods, it isn't functioning properly; particularly, the addsongs method is not working as it should.
function addsongs(playlist_object, id_entered){
var arraysongs = [];
models.Playlist.findOne({
attributes: ['id'],
where: {
id: id_entered
}
})
.then(function(playlist) {
playlist.getSongs().then(function (thesongs){
for(var k = 0; k < thesongs.length ; k++){
arraysongs.push(thesongs[k].Songs_Playlists.SongId);
}
playlist_object.playlists[(id_entered - 1)]['songs'] = arraysongs;
});
});
}
I cannot for the life of me figure out why it works when the top segment of code is at the top, but doesn't work when inside my app.get() call.

From your code I have conducted that you want to return playlists (id and name) together with their songs (id). First of all your code will not work because the calls of addsongs(data_playlists, id) are run before data_playlists is filled with data by code above it. Moreover, the addsongs function performs asynchronous operations returning Promises, so calling them one by one will not give expected result. I suppose you can do it completely differently.
I suggest you use include attribute of options object that can be passed to findAll() method. include says which association model you also want to return from current query. In this case you want to return playlists together with their songs (M:M relation according to your code), so you need to include Song model in the query.
function getPlaylistsWithSongs() {
return models.Playlist.findAll({
attributes: ['id', 'name'],
include: [
{
model: models.Song,
as: 'Songs', // depends on how you have declare the association between songs and playlists
attributes: ['id'],
through: { attributes: [] } // prevents returning fields from join table
}
]
}).then((playlistsWithSongs) => {
return playlistsWithSongs;
});
}
Example result of getPlaylistsWithSongs result would be (after translating it to JSON e.g. like playlistsWithSongs.toJSON())
[
{
id: 1,
name: 'playlist #1',
Songs: [
{ id: 1 },
{ id: 2 }
]
}
]
Above code returns all playlists (their id and name) with their songs (only their id). Now in your route resolver you can simply call above function to return the result
app.get('/api/playlists', function (request, response) {
response.setHeader("Content-Type", "application/json; charset=UTF-8");
getPlaylistsWithSongs().then(function(playlistsWithSongs){
response.status(200).send(JSON.stringify(playlistsWithSongs));
});
});
EDIT
In order to simply return array of IDs instead array of objects with id (songs), you need to map the result. There is no simple sequelize way to return array of IDs in such a case.
}).then((playlistWithSongs) => {
let jsonPlaylists = playlistsWithSongs.map((singlePlaylist) => {
// return JSON representation of each playlist record
return singlePlaylist.toJSON();
});
jsonPlaylists.forEach((playlist) => {
// at every playlist record we map Songs to array of primitive numbers representing it's IDs
playlist.songs = playlist.Songs.map((song) => {
return song.id;
});
// when we finish we can delete the Songs property because now we have songs instead
delete playlist.Songs;
});
console.log(jsonPlaylists);
// example output: [{ id: 1, name: 'playlist #1', songs: [1, 2, 3] }]
return jsonPlaylists;
});

Meteor, query Collections based on ids retrieved from another collection

I have 2 collections.
ItemList = new Mongo.Collection('items');
BorrowerDetails = new Mongo.Collection('borrow');
ItemList.insert({
brand: "brand-Name",
type: "brand-Type",
._id: id
});
BorrowerDetails.insert({
key: "ItemList.id", //equals to .id of the ItemList Collection
name : "borrowerName"
});
Question !
How can i retrieve records from the BorrowerDetails Collection based on a certain type from the ItemList Collection.
ex. Retrieve all records from the BorrowerDetails Collection where key is equals to the id of a record on the ItemList Collection whose type is equals to "Desktop".
return BorrowerDetails.find(
{ key :
ItemList.find(
{ type : 'Desktop' },
{ fields: {'_id':1 } }
)
}
); //error!

Note that I don't have nodejs right now in my laptop, so might have several errors since I am not able to test it.
First, Create a publication file (eg. server\publications\borrowPub.js). inside the file, you should create a publication method. The logic here is simple, get the itemid array first, and then pass it as parameter in Mongo select $in.
Meteor.publish('queryBorrowerTypeDesktop', function(criteria)
{
var itemArr = ItemList.find( { type : 'Desktop' },
{ fields: {'_id':1 } });
if(itemArr){
//itemArr found, so do a select in using the array of item id we retrieved earlier
var borrowers = BorrowerDetails.find({ key: { $in: itemArr } });
return borrowers;
}else{
//found nothing- so return nothing
return []
}
});
Second, in the router :
Router.route('/test', {
name: 'test',
action: function()
{
//change accordingly.
},
waitOn: function()
{//subscribe from publisher that we just created...
return [
Meteor.subscribe('queryBorrowerTypeDesktop')
];
},
data: function() {
if(Meteor.userId())
{
// also include the sorting and limit so your page will not crash. change accordingly.
return {
borrowerDetails: BorrowerDetails.find({},{sort: {name: -1}, limit: 30}),
}
}
}
});
Note that in data, BorrowerDetails.find() does not need to filter by anything because it has been filtered during the subscribe, which has been cached in MiniMongo in your browser.

Got duplicated data when subscribe multiple times

I am using MongoDB aggregation in meteor. I got duplicated data when subscribe multiple times.
(The data in database are static, which means they are same all the time.)
// Server side
Meteor.publish('totalNumber', function () {
let pipeline = [
{ $unwind: '$product' },
{ $group: {
_id: {
code: '$product.code',
hour: { $hour: '$timestamp' }
},
total: { $sum: '$product.count' },
}}
];
Products.aggregate(
pipeline,
Meteor.bindEnvironment((err, result) => {
console.log('result', result); // at here every time subscribe, no duplicated data
_.each(result, r => {
this.added('totalNumber',
// I use Random.id() here, because "Meteor does not currently support objects other than ObjectID as ids"
Random.id(), {
code: r._id.code,
hour: r._id.hour,
total: r.total
});
});
}
)
);
this.ready();
});
// Client side
this.subscribe('totalNumber', () => {
// Correct result: [Object, Object] for example
console.log(Products.find().fetch());
}, true);
this.subscribe('totalNumber', () => {
// Wrong result: [Object, Object, Object, Object]
console.log(Products.find().fetch());
}, true);
this.subscribe('totalNumber', () => {
// Wrong result: [Object, Object, Object, Object, Object, Object]
console.log(Products.find().fetch());
}, true);
So right now basically, the new results always include last time subscribe data.
How can I solve this problem? Thanks

The problem is that you are using a random id each time in the call to added so the client always thinks all of the documents are unique. You need to devise a consistent id string generator. Using an answer to this question, you could imagine building a set of functions like these:
hashCode = function (s) {
return s.split('').reduce(function (a, b) {
a = ((a << 5) - a) + b.charCodeAt(0);return a & a;
}, 0);
};
objectToHash = function (obj) {
return String(hashCode(JSON.stringify(obj)));
};
So if you wanted a unique document for each combination of code and hour you could do this:
var id = objectToHash(r._id);
this.added('totalNumber', id, {...});

Recursive include Sequelize?

I have category that can have child categories
And when I'm doing findAll I want to include all of those nested, but I don't know the depth.
var includeCondition = {
include: [
{
model: models.categories,
as:'subcategory', nested: true
}]
};
models.categories.findAll(includeCondition)
.then(function (categories) {
resolve(categories);
})
.catch(function (err) {
reject(err);
})
});
The result brings me only one level nested include.
[
{
dataValues:{
},
subcategory:{
model:{
dataValues:{
}
// no subcategory here
}
}
}
]
Can I somehow make sequalize include those nested subcategories ?

There are few solutions if found for this
first one is more complicated but will give better performance:
This one is about implementing hierarchical data structure in MySQL
I like the guide here
http://mikehillyer.com/articles/managing-hierarchical-data-in-mysql/
The one that is named The Nested Set Model.
The second solution that I actually implemented by myself is recursive expanding, this one uses lots of mysql requests and I believe can be improved, but it's a fast one and works well. The thing is to use for each category function like this
var expandSubcategories = function (category) {
return new promise(function (resolve, reject) {
category.getSubcategories().then(function (subcategories) {
//if has subcategories expand recursively inner subcategories
if (subcategories && subcategories.length > 0) {
var expandPromises = [];
_.each(subcategories, function (subcategory) {
expandPromises.push(expandSubcategories(subcategory));
});
promise.all(expandPromises).then(function (expandedCategories) {
category.subcategories = [];
_.each(expandedCategories, function (expandedCategory) {
category.subcategories.push(expandedCategory);
}, this);
//return self with expanded inner
resolve(category);
});
} else {
//if has no subcategories return self
resolve(category);
}
});
});
};
So it's going through the categories and expanding them recursively.
Maybe this will help someone as well.

This is ihoryam's answer adapted to ES6, using async/await, arrow functions () => and Sequelize ORM to fetch the data, and not using Lodash.
const getSubCategoriesRecursive = async (category) => {
let subCategories = await models.category.findAll({
where: {
parentId: category.id
},
raw : true
});
if (subCategories.length > 0) {
const promises = [];
subCategories.forEach(category => {
promises.push(getSubCategoriesRecursive(category));
});
category['subCategories'] = await Promise.all(promises);
}
else category['subCategories'] = [];
return category;
};
Async functions returning promises, you do not need to precise return new promise(...)

There is a node module which handle it : sequelize-hierarchy
It adds column parentId and hierarchyLevel to your table.
As an example, this is what I did to order employees skills in a tree.
Skills could be "Macro" -> "Excel" -> "Office" -> "Computer"
database.js:
const Sequelize = require('sequelize');
require('sequelize-hierarchy')(Sequelize);
const sequelize = new Sequelize("stackoverflow", null, null, {
dialect: "sqlite",
storage: "database.db"
});
sequelize.sync().then(() => {console.log("Database ready");});
module.exports = sequelize;
skill.js:
module.exports = (sequelize, DataTypes) => {
const Skill = sequelize.define("skill", {
name: DataTypes.STRING,
});
Skill.isHierarchy();
return Skill;
};
Then in your controller:
Skill.findAll().then(skills => {
res.send(skills); // Return a list
});
Skill.findAll({ hierarchy: true }).then(skills => {
res.send(skills); // Return a tree
});

Sequelize currently has no support for common table expressions and recursive CTEs. Adding the ability to include a CTE into the find* family of methods would allow find* to perform recursive queries.
Here is the link for examples.
Common Table Expressions and Recursive Queries

Suppose you have 5 different models A, B, C, D, E and A is associated with B, B with C and so on.
So while fetching data for A you can get the all the nested subcategory hierarchy by using
include: [{ all: true, nested: true }]
Example:
A.findAll(where:{// add conditions}, { include: [{ all: true, nested: true }]});