Looking at https://github.com/vitaly-t/pg-promise/wiki/Data-Imports there's a very detailed doc on how to use it for importing.
However while that works for the demoed scenario I don't know how to apply it on my case.
When I do my web call, I get the actual JSON data and a paramter in the header which gives me a value for the next page (could be a date or String or a number value).
In the example, it says:
db.tx('massive-insert', t => {
return t.sequence(index => {
return getNextData(index)
.then(data => {
if (data) {
const insert = pgp.helpers.insert(data, cs);
return t.none(insert);
}
});
});
})
.then(data => {
console.log('Total batches:', data.total, ', Duration:', data.duration);
})
.catch(error => {
console.log(error);
});
In this case, sequence(index will use index which seems to increment +1.
But in my case,
function getNextData(nextPage) {
//get the data for nextPage
.....
//get the nextPage if exists for future use
nextPage = response.next;
resolve(data);
}
My question is, how can I replace index with nextPage in this example, as each new Promise needs to use the nextPage from previous one.
LATER EDIT: And if I want to fetch info from a certain value of nextPageInfo?
For instance:
db.any('Select value from table')
.then(function(value) {
var data = value; //not working
db.tx('massive-insert', t => {
return t.sequence((index, data) => {
return getNextData(index, data)
.then(a => {
if (a) {
const insert = pgp.helpers.insert(a.data, cs);
return t.none(insert).then(() => a.nextPageInfo);
}
})
});
})
.then(data => {
// COMMIT has been executed
console.log('Total batches:', data.total, ', Duration:', data.duration);
})
.catch(error => {
// ROLLBACK has been executed
console.log(error);
})
}
Following this question, I have extended article Data Imports with the new extras section, which gives you exactly the example that you need. The example copied from the article:
function getNextData(t, index, nextPageInfo) {
// t = database transaction protocol
// NOTE: nextPageInfo = undefined when index = 0
return new Promise((resolve, reject) {
/* pull the next data, according to nextPageInfo */
/* do reject(error) on an error, to ROLLBACK changes */
if(/* there is still data left*/) {
// if whateverNextDetails = undefined, the data will be inserted,
// but the sequence will end there (as success).
resolve({data, nextPageInfo: whateverNextDetails});
} else {
resolve(null);
}
});
}
db.tx('massive-insert', t => {
return t.sequence((index, data) => {
return getNextData(t, index, data)
.then(a => {
if (a) {
const insert = pgp.helpers.insert(a.data, cs);
return t.none(insert).then(() => a.nextPageInfo);
}
})
});
})
.then(data => {
// COMMIT has been executed
console.log('Total batches:', data.total, ', Duration:', data.duration);
})
.catch(error => {
// ROLLBACK has been executed
console.log(error);
});
Please note that since we are chaining the result from getNextData to the value of nextPageInfo, then if its value is undefined, it will do the next insert, but then will end the sequence (as success).
Related
I'm using mysql npm package to get data from the database.
My goal is to add every list(id) from lists Table and push it into lists array.
I'm getting the correct data from the database but when I result the query lists array is empty.
I think that I have to add async and await to the code to make it work. Tried in several places but I didn't make it work. Any idea what I'm doing wrong?
// GET - get all grocery_lists
Grocery_list.getAll = (result) => {
let lists = []; // <--- List Array
sql.query("SELECT id FROM lists", (err, res) => { // <--- Get all id from 'lists' table
if (err) {
console.log(err);
}
res.forEach(list => { // <--- Loop thru all lists
sql.query(`
SELECT items.id, items.name, items_x_lists.purchased
FROM items_x_lists
INNER JOIN items ON items_x_lists.itemId = items.id
WHERE items_x_lists.listId = ${list.id};
`, (err, res) => { // <--- Get all items for ${list.id} from 'items' table
if (err) {
console.log(err);
}
const list = {};
list.id = res.id;
console.log(list); // <--- { id: 1 } ... { id: 2 }
lists.push(list);
});
});
result(null, lists); // <--- returning empty array instead of [{ id: 1 }, { id: 2 }]
});
};
I think you can do it simply like this (not the exact code) but I think you've got the idea:
// GET - get all grocery_lists
Grocery_list.getAll = (result) => {
getData().then(data => {
// call result(data);
}).catch(err => {
// handle error for no data or any other errors
})
}
const getData = async () => {
try {
var res = await sql.query("SELECT id FROM lists");
// Handle error if no data found
if (!res) { Promise.reject("No data found"); }
} catch (error) {
return Promise.reject(error);
}
const listIds = res.map(id => id); // array of ids
try {
var data = await sql.query(`
SELECT items.id, items.name, items_x_lists.purchased
FROM items_x_lists
INNER JOIN items ON items_x_lists.itemId = items.id
WHERE items_x_lists.listId in ${listIds};`);
// Handle error if no data found
if (!data) { return Promise.reject("No data found") }
} catch (error) {
return Promise.reject(error);
}
return Promise.resolve(data)
}
this mighht be the solution
Grocery_list.getAll = async (result) => { return sql.query("SELECT id FROM lists", (err, res) => { // <--- Get all id from 'lists' table if (err) { console.log(err); } }); };
I’m outside, so I can only edit with my mobile phone. If the layout is strange, I apologize for it.
First of all, if it is me, I will adjust a few places
1.Create an async function and use Promise.all technology
async function queryAll(querys){
return await Promise.all(querys);
}
2.Create a Promise function to execute each sql
const queryPromise = id => {
return new Promise((resolve, reject) => {
sql.query(
'SELECT items.id, items.name, items_x_lists.purchased FROM items_x_lists INNER JOIN items ON items_x_lists.itemId = items.id WHERE items_x_lists.listId = ? ;',
id,
(err, rows, fields) => {
console.log(rows);
if (err) reject(err);
else resolve(rows);
}
);
});
};
3.Adjust the internal logic of the getAll event
const querys = []
res.forEach(list => {
querys.push(queryPromise(list.id));
});
const querysResults = await queryAll(querys);// remember to use async for function
querysResults.forEach(querysResult => {
lists.push({id:querysResult['id']});
});
Because there is no computer on hand, there may be some situations that need to be judged, but roughly this will work normally.
Hope this helps you :)
Update: Refactored the code at the end below to this:
let getMembersIDs = eventID => {
return admin.firestore().collection('events').doc(eventID).get()
.then(doc => {
if (doc.exists) {
console.log(doc.data().members)
return doc.data().members
}
else throw new Error ('Event does not exist!', doc.id)
})
}
let dispatchToMembers = (arrayMembersIDs, data) => {
return Promise.all(
arrayMembersIDs.map(memberID => {
if (data.at===memberID) data.type = 'pm'
else data.content = 'New message in an event chat you participate to.'
console.log('Sending now to: ', memberID)
return admin.firestore().collection('users').doc(memberID).collection('inbox').add({
content: 'You were mentioned in a recent chat: ' + data.content,
type: data.type
})
})
)
}
getMembersIDs(data.target).then(members => dispatchToMembers(members, data)).then(() => {
console.log('Message dispatched!')
res.end()
return true
})
.catch(err => {
console.error(err);
response.status(500).send(err);
return true;
})
})
It does work when I run it from my project, replacing admin.firestore() with db. However when embedded within a cloud function it does not work: the cloud function returns codes 204 and 200 but the desired db operation does not seem to occur. Pulling my hair as I don't understand why.
Outdate: I am not able to debug the following piece of code. A simple cloud function with two parts (a read, a write) chained with promises.
exports.discuss = functions.https.onCall((data, context) => {
return admin.firestore().collection('events').doc(data.target).get()
.then(doc => {
if (doc.exists) return doc.data().members
else throw new Error ('Doc does not exist ', doc.id)
})
.then(members => {
let promises = []
members.forEach(u => {
...
let promise = new Promise(res => {
admin.firestore().collection('users').doc(u).collection('inbox').add({...})
res(u)
})
promises.push(promise)
})
return Promise.all(promises)
})
.then(() => {
response.send('ok')
return;
})
.catch(err, response.status(500).send(err))
})
You generate that 500 code with this line:
.catch(err, response.status(500).send(err))
The client that invokes the Cloud Function should be getting the error message in the response body. But you'll also want to log it on the server with:
.catch(function(err) {
console.error(err);
response.status(500).send(err);
return true;
})
I'm pulling category information from our local point of sale database (3rd party software) and trying to write it into a WooCommerce store. I'm also storing the data to my own database to maintain relationships between the two systems. I'm using promises to sequence everything, but one of my .then() statements is firing off before the previous .then() has returned, so I'm sending an empty payload to WooCommerce.
router.post("/:action", (req, res) => {
if(req.params.action === "sync" && req.body.action === "sync") {
// Query the POS database
mssql.query(query, (err, result) => {
let postData = {
create: [],
update: []
}
// Make some promises to pull the POS categories and their children
Promise.all(promises)
.then(cats => {
let catPromises = cats.map(cat => {
return new Promise((resolve, reject) => {
Category.findOne(
// Check for existing entry in the linking DB...
)
.then(data => {
// ...and handle accordingly
resolve()
})
.then(() => {
let childPromises = cat.children.map(child => {
return new Promise((resolve, reject) => {
Category.findOne(
// Checking for existing entry in the linking DB...
)
.then(data => {
// ...and handle accordingly
resolve()
})
})
})
Promise.all(childPromises)
.then(resolved => {
resolve()
})
})
})
})
Promise.all(catPromises)
.then(() => {
return
})
})
.then(() => {
// This is the part that's firing early
return axios.post(
// data
)
})
...
EDIT: Newly refactored, still having problems.
Promise.all(promises).then(cats => {
let catPromises = cats.map(cat => {
Category.findOne(
// Check for existing...
).then(data => {
// ...and handle accordingly
}).then(() => {
let childPromises = cat.children.map(child => {
Category.findOne(
// Check for existing...
).then(data => {
// ...and handle accordingly
})
})
return Promise.all(childPromises)
})
})
// Now this is where we're reaching early
return Promise.all(catPromises)
}).then(() => {
// API call
})
Final solution:
Promise.all(promises).then(cats => {
let catPromises = cats.map(cat => {
return Category.findOne(
// Check for existing...
).then(data => {
// ...and handle accordingly
}).then(() => {
let childPromises = cat.children.map(child => {
return Category.findOne(
// Check for existing...
).then(data => {
// ...and handle accordingly
})
})
return Promise.all(childPromises)
})
})
// Now this is where we're reaching early
return Promise.all(catPromises)
}).then(() => {
// API call
})
My script is throwing the following error when returning the result of the assync firestore set function:
You must return a Promise in your transaction()-callback.
According to firebase documentation about transactions, set function return a transaction itself.
Here a simplified copy of my code.
var myDoc = {
field1: "v1"
};
var docRef = db
.collection("docs")
.doc("1");
return db
.runTransaction(t => {
return t
.set(docRef, chat, {merge:false}); //has i understand, this should return a transaction object but the error says otherwise.
})
.then( doc => {
response.send();
})
.catch(err => {
...;
})
I am still new to Nodejs and not very familiar with chaining assyncs methods, so i must be doing some obvious error here.
Haven't used firestore transaction but I have used firebase transactions. You can try following
return db
.runTransaction(t => {
return t.set(docRef, chat, {merge:false})
.then(data => {
return Promise.resolve('transaction complete');
})
.then( doc => {
response.send();
})
.catch(err => {
...;
})
and the method that encloses your whole code must be returning promise as you have written return db.runTransaction(t => {....})
so if that is not needed then use
var transaction = db.runTransaction(t => {...});
Just do this:
return db
.runTransaction(t => {
t.set(docRef, chat, {merge:false});
return Promise.resolve(); // Add this line.
})
.then( doc => {
response.send();
})
.catch(err => {
...;
})
If the t.set() failed, it won't go to the resolve() anyway.
The following code loops through some form fields. If the field is a file that has to be uploaded it runs an api.uploadPhotofunction (setting the payload once the photos has been uploaded). If the field is a normal input when the payload is set directly:
formFields.forEach(field => {
if (hasUploadFiles(field)) {
uploadPhotoPromise = new Promise((resolve, reject) => {
uploads.queued.push(file)
api.uploadPhoto(file, field).then(uploadedPhoto => {
uploads.finished.push(field)
if (uploads.queued.length === uploads.finished.length) {
payload[field.name] = uploadedPhoto
resolve()
} else {
reject()
}
}).catch(error => {
console.log('error:', error)
reject()
})
}).catch(error => {
console.log('error:', error)
})
} else {
payload[field.name] = field.value
}
})
Promise.all([uploadPhotoPromise]).then(values => {
// update action
}
The code works. However, all those catch make it look a bit messy.
I tried removed them but the code hangs if I remove any of them (the code inside Promise.all never runs). Why is this? And how to refactor this code without all those catch statements without making the it hang?
Original code (plus Bergi's suggested modification):
const buildingFormPromise = utils.mapDeep(this.buildingForm.schema, field => {
if (!field.name) return // fields not in the database
else if (utils.hasUploadFiles(field)) {
utils.eachCall(field.value, (file, index) => {
field.userId = this.user.id
this.uploads.queued.push(file)
this.$set(this.uploads.queued, index, { progress: 30 })
return api.uploadPhoto(file, field).then(uploadedPhoto => {
this.$set(this.uploads.queued, index, { progress: 100 })
return loadImage(uploadedPhoto, () => {
this.uploads.finished.push(field)
if (this.uploads.queued.length === this.uploads.finished.length) {
console.log('This runs after the code inside Promise.all')
buildingPayload[field.name] = uploadedPhoto
}
})
})
})
} else {
return Promise.resolve(buildingPayload[field.name] = field.value)
}
})
Promise.all([buildingFormPromise]).then(values => {
console.log('This runs before the files are uploaded')
})
You need to pass an array of all the promises into Promise.all, and you should avoid the Promise constructor antipattern. You can move the .catch to the very end if you don't want to handle individual upload failures.
var fieldValuePromises = formFields.map(field => {
if (hasUploadFiles(field)) {
return api.uploadPhoto(file, field).then(uploadedPhoto => {
return payload[field.name] = uploadedPhoto;
});
} else {
return Promise.resolve(payload[field.name] = field.value);
}
});
Promise.all(fieldValuePromises).then(values => {
// update action
}).catch(error => {
// at least one upload failed
console.log('error:', error)
});