I am making an Api with Node and Mongo that receives large volumes of data, I was receiving an error because the size of the records that were stored in mongo exceeded 16 MB. So I opted for the alternative offered by mongo in its gridFS documentation, to insert the records, which I had no problems with. But I am having conflicts to insert and filter since I don't know how to do it, I read the documentation and there are several ways. But I can't figure out how to filter (find a record by its field) and how to update.
The function to create a record works but it performs some necessary steps such as storing the json it receives in a file and then reading it and with that creating the record, I would have liked to find a more practical solution such as only inserting the json it receives without having to create a file with its content and then get the information from that file I attach the code to see if you can tell me how to solve this problem:
const { MongoClient, ObjectId, GridFSBucket,} = require('mongodb');
const { config } = require('../../config');
//const USER = encodeURIComponent(config.noRelDbUser);
//const PASSWORD = encodeURIComponent(config.noRelDbPassword);
const DB_NAME = config.noRelDbName;
const fs = require('fs');
const removeFile = require('../../modules/results/utils/RemoveFile');
// const MONGO_URI = `mongodb://${USER}:${PASSWORD}#${config.dbHost}:${config.dbPort}/admin?retryWrites=true&w=majority`
const MONGO_URI = `mongodb://${config.noRelDbHost}:${config.noRelDbPort}`;
class MongoLib {
constructor() {
this.client = new MongoClient(MONGO_URI, { useNewUrlParser: true, useUnifiedTopology: true });
this.dbName = DB_NAME;
}
connect() {
if (!MongoLib.connection) {
MongoLib.connection = new Promise((resolve, reject) => {
this.client.connect((err) => {
if (err) {
reject(err);
}
resolve(this.client.db(this.dbName));
});
});
}
return MongoLib.connection;
}
create(collection, data) {
return this.connect()
.then((db) => {
return db.collection(collection).insertOne(data);
})
.then((result) => result.insertedId);
}
async createWithForBigData(collection, data, vr_id , remove=false){
let vrule_id = vr_id;
return this.connect().then((db)=>{
try{
var bucket = new GridFSBucket(db, {
bucketName: collection,
chunkSizeBytes: 260000 ,
});
bucket
bucket.find()
let uploadStream = fs.createReadStream(data).pipe(bucket.openUploadStream(`resultsdetail${vrule_id}`));
let id = uploadStream.id;
uploadStream.on('error', (err) => {
console.log({ message: "Error uploading file" });
throw new Error(err);
});
bucket.find()
uploadStream.on('finish', () => {
console.log({ message: "File uploaded successfully, stored under Mongo ObjectID: " + id });
if(remove === true){
console.log('remueve archivo archivo de directorio storebigdata');
removeFile(data);
}
return id;
});
}catch(err){
console.log('ocurrió un error al almacenar big data',err);
throw new Error(err);
}
})
}
findBigData(){
//
}
UpdateBigData(){
//
}
}
module.exports = MongoLib;
I would like to know, how can I pass a field value of a QueryResolver to another JS File (database.js) in GraphQL and NodeJS/Apollo. In the database.js file I connect with my MYSQL Database with MariaDB and get dynamically data information. The information should be get by the ID which the user enters in the query. The id is know in my QueryResolver but I don't know how I can pass this ID to my database.js file, so that my user gets the data that he wants.
Instead of the 0 in the code, there should be the ID the user entered.
What I tried:
I tried to save the ID in a variable and to return the id AND the articledatas ( these are my dynamic datas the user gets after the query ). And I tried to get access to the function get_result for getting the id, but then I get stuck, because I don't know I could access to this function outside the query. Could anyone help me please?
My resolver looks like this (this is my problem):
module.exports = {
Query: {
article: (parent, {id}) => {
var data = models.articledata.find((c) => c.id == id);
var id_work = id;
function get_returns(){return [data, id_work];}
var get_results = get_returns();
return get_results[0];
},
},
Node: {
__resolveType(node) {
if(node.toObject().name){
return 'Article';
}
}
}
}
This is my data.js (in Models folder):
//This works fine!
var read = function(){
return new Promise(function(resolve, reject){
var query_str = 'SELECT * FROM Artikel';
conn.query(query_str, function (err, rows, fields) {
if (err) {
return reject(err);
}
resolve(rows)
});
});
}
//Here I want to replace the 0 with the {id} of the resolver above->Problem how can I get a reference of the {id} from the resolver?
var title = read().then(function(rows){return rows[0].NAME});
var articledatas = [{
title: title,
}];
module.exports = { articledatas };
```
This question is a duplicate of How to return the response from an asynchronous call in most of the time. Please read the answer to the question I refer.
After some tries I figured out when I past the code of my data.js DIRECTLY into the Query, then I don't have to pass the ID outside the Query. This works:
module.exports = {
Query: {
article: (parent, {id}) => {
var read = function(){
return new Promise(function(resolve, reject){
var query_str = 'SELECT * FROM Artikel';
conn.query(query_str, function (err, rows, fields) {
if (err) {
return reject(err);
}
resolve(rows)
});
});
}
var title = read().then(function(rows){return rows[id].NAME});
var articledatas = [{
title: title,
}];
return = articledata.find((c) => c.id == id);
},
},
Node: {
__resolveType(node) {
if(node.toObject().name){
return 'Article';
}
}
}
}
It seems im using async wrong, can anybody spot what I am doing wrong?
This is the function I am waiting on:
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
export async function firebaseAcceptTradeOffer(tradeOfferID, userData) {
var tradeInstanceID;
var senderID;
var receiverID;
var senderItemsTemp;
var receiverItemsTemp;
var response;
var tradeOffer = db.collection("tradeOffers").doc(tradeOfferID);
return tradeOffer
.get()
.then((doc) => {
senderItemsTemp = doc.data().sendersItems;
receiverItemsTemp = doc.data().receiversItems;
senderID = doc.data().senderID;
receiverID = doc.data().receiverID;
})
.then(() => {
var itemInTrade = false;
senderItemsTemp.forEach((item) => {
db.collection("listings")
.doc(item.itemID)
.get()
.then((doc) => {
if (doc.data().status !== "listed") {
itemInTrade = true;
}
})
.then(() => {
receiverItemsTemp.forEach((item) => {
db.collection("listings")
.doc(item.itemID)
.get()
.then((doc) => {
if (doc.data().status !== "listed") {
itemInTrade = true;
}
})
.then(() => {
if (itemInTrade) {
tradeOffer.update({
status: "declined",
});
return false;
} else {
db.collection("trades")
.add({
tradeOfferID: tradeOfferID,
senderTradeStatus: {
created: true,
sentToSeekio: "current",
inspection: false,
sentToPartner: false,
},
receiverTradeStatus: {
created: true,
sentToSeekio: "current",
inspection: false,
sentToPartner: false,
},
postagePhotos: [],
inspectionPhotos: [],
senderPaid: false,
receiverPaid: false,
senderUploadedProof: false,
receiverUploadedProof: false,
senderID: senderID,
receiverID: receiverID,
messages: [
{
message: `Trade created. A representative, will message this chat shortly with instructions and postage address. If you would like more information about the trading process, head to seekio.io/help. Thank you for using Seekio!`,
sender: "System",
timestamp: firebase.firestore.Timestamp.fromDate(
new Date()
),
},
],
})
.then((docRef) => {
tradeInstanceID = docRef.id;
tradeOffer
.set(
{
status: "accepted",
tradeInstanceID: docRef.id,
},
{ merge: true }
)
.then(() => {
var receiver = db.collection("users").doc(senderID);
var notification = {
from: auth.currentUser.uid,
fromUsername: userData.username,
type: "tradeOfferAccepted",
time: firebase.firestore.Timestamp.fromDate(
new Date()
),
seen: false,
};
receiver
.update({
notifications: firebase.firestore.FieldValue.arrayUnion(
notification
),
})
.then(() => {
response = {
sendersItems: senderItemsTemp,
receiversItems: receiverItemsTemp,
};
return response;
});
});
})
.catch((err) => console.log(err));
}
});
});
});
});
});
}
And here is where I am calling it:
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
async function acceptTradeOffer() {
var tradeOfferID = currentTradeFocus;
var senderID = "";
setLoading("loading");
if (userData !== null && tradeOfferID !== "") {
const response = await firebaseAcceptTradeOffer(
currentTradeFocus,
userData
);
console.log(
"RESPONSE FROM FIREBASE SERVICE>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>: ",
response
);
if (!response) {
setErrorMsg("One of the selected items is no longer available.");
} else if (
response.sendersItems !== null &&
response.receiversItems !== null
) {
setSenderItems(response.sendersItems);
setReceiverItems(response.receiversItems);
toggleConfirmScreen("cancel");
setLoading("idle");
setItemsSet(true);
}
fetch(
"https://europe-west2-seekio-86408.cloudfunctions.net/sendMail?type=tradeUpdate&userID=" +
senderID
).catch((err) => {
console.log(err);
setLoading("idle");
});
}
}
So basically I want to go and check if any of the items in this 'trade' are not equal to 'listed' (which means they are not available, I want to return false, if not, then I return the array of items so the trade can continue.
EDIT: I've tried to rejig it all and it's half working. A top level look at what I am trying to do:
User wants to accept a trade offer for some items >
Check through all items to make sure they are available and not sold >
If so, accept the trade >
Then once its accepted, go and cancel all remaining trade offers that include items from this accepted trade, cause they are not available anymore.
//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
export async function firebaseAcceptTradeOffer(tradeOfferID, userData) {
console.log(
"----- starting firebaseAcceptTradeOffer--------- ",
unavailableItem
);
//==============
var tradeInstanceID;
var senderID;
var receiverID;
var senderItemsTemp;
var receiverItemsTemp;
var unavailableItem = false;
var response;
var itemsArray;
var notListed = false;
//==============
var tradeOffer = db.collection("tradeOffers").doc(tradeOfferID);
unavailableItem = tradeOffer
.get()
.then((doc) => {
senderID = doc.data().senderID;
receiverID = doc.data().receiverID;
itemsArray = doc.data().sendersItems.concat(doc.data().receiversItems);
})
.then(() => {
itemsArray.forEach((item) => {
db.collection("listings")
.doc(item.itemID)
.get()
.then((doc) => {
if (doc.data().status !== "listed") {
notListed = true;
}
});
});
})
.then(() => {
return notListed;
});
console.log(
"-----unavailableItem at the end of method --------- ",
unavailableItem
);
//^^^^^^^^^^ here i am getting a promise result of false (which is correct) but HOW CAN I ACCESS IT
if (unavailableItem) {
tradeOffer.update({
status: "declined",
});
return false;
} else {
response = await createTrade(
tradeOffer,
tradeOfferID,
senderID,
receiverID,
userData.username
);
console.log("response from createTrade", response);
return response;
}
}
I am getting a promise object back with the value false above. False is correct value I am expecting, but how can I access it? its in the form of a promise object?
I have some time on my hands, so let's break this down.
Notes on Variables
If you aren't using TypeScript (and even if you are), I highly recommend inserting the type into the name of your variables.
db # ✔ by convention, either firebase.database() or firebase.firestore()
tradeOffer # ❓ type unclear, could be a number, an object, a string, etc
tradeOfferDocRef # ✔ a DocumentReference
trades # ❓ type unclear, plural implies a collection of some sort
tradesColRef # ✔ a CollectionReference
You may also encounter these:
doc # ❓ by convention, a DocumentSnapshot, but with unknown data
tradeDoc # ✔ implies a DocumentSnapshot<TradeData> (DocumentSnapshot containing trade data)
When using just doc, you need to look around where its used for context on what this DocumentSnapshot contains.
db.collection('trades').doc(tradeOfferID).get()
.then((doc) => { // contents implied to be TradeData
const data = doc.data();
});
// or
tradeDocRef.get()
.then((doc) => { // contents implied to be TradeData
const data = doc.data();
});
You should rename doc as appropriate, especially when using async/await syntax, so you don't end up in situations like:
const doc = await db.collection('trades').doc(tradeOfferID).get();
/* ... many lines ... */
const senderID = doc.get("senderID"); // what was doc again?
As you've tagged reactjs in your question, this implies you are using modern JavaScript.
Ditch any use of var and replace it with the block-scoped versions: const (prevents reassigning the variable) or let (similar to var, but not quite). These are safer and prevents the chances of accidentally overwriting something you shouldn't.
You can also make use of Object destructuring to assign your variables.
const senderID = doc.data().senderID;
const receiverID = doc.data().receiverID;
const itemsArray = doc.data().sendersItems.concat(doc.data().receiversItems);
can become:
const { senderID, receiverID, sendersItems, receiversItems } = doc.data();
const itemsArray = sendersItems.concat(receiversItems);
If you ever need only a single property out of a document, you should use DocumentSnapshot#get() instead of DocumentSnapshot#data() so it will parse only the field you want instead of the whole document's data.
function getUserAddress(uid) {
return firebase.firestore()
.collection('users')
.doc(uid)
.get()
.then(userDoc => userDoc.get("address")); // skips username, email, phone, etc
}
Notes on Promises
var senderID;
var receiverID;
var itemsArray;
tradeOfferDocRef
.get()
.then((doc) => {
senderID = doc.data().senderID;
receiverID = doc.data().receiverID;
itemsArray = doc.data().sendersItems.concat(doc.data().receiversItems);
})
.then(() => {
/* use results from above */
});
While the above code block functions as intended, when you have many of these variables like this as you do, it becomes unclear when and where they are set.
It also leads to problems like this where you think the variable has a value:
var senderID;
var receiverID;
var itemsArray;
tradeOfferDocRef
.get()
.then((doc) => {
// this line runs after the line below
senderID = doc.data().senderID;
receiverID = doc.data().receiverID;
itemsArray = doc.data().sendersItems.concat(doc.data().receiversItems);
});
// this line before the line above
console.log(senderID); // will always log "undefined"
This can be avoided in one of three ways:
Returning data to pass through to the next handler (you wouldn't use this for this example, only if the next then() handler is elsewhere):
tradeOfferDocRef
.get()
.then((doc) => {
const { senderID, receiverID, sendersItems, receiversItems } = doc.data();
const itemsArray = sendersItems.concat(receiversItems);
return { senderID, receiverID, itemsArray }; // pass to next step
})
.then((neededData) =>
/* use neededData.senderID, neededData.receiverID, etc */
});
Using the data within the same handler:
tradeOfferDocRef
.get()
.then((doc) => {
const { senderID, receiverID, sendersItems, receiversItems } = doc.data();
const itemsArray = sendersItems.concat(receiversItems);
/* use results from above */
});
Using async-await syntax:
const tradeDoc = await tradeOfferDocRef.get();
const { senderID, receiverID, sendersItems, receiversItems } = tradeDoc.data();
const itemsArray = sendersItems.concat(receiversItems);
/* use results from above */
Writing to Firestore
Your current code consists of the following steps:
1. Get the trade offer document</li>
2. If successful, pull out the sender and receiver's IDs, along with any items in the trade
3. If successful, do the following for each item in the sender items array:
a) Check if any of the sender's items are unavailable</li>
b) If successful, do the following for each item in the receiver items array:
- If **any item** was unavailable prior to this, decline the trade & return `false`.
- If all items **so far** are available, do the following:
a) Create a document containing information about the trade with the needed data
b) If successful, edit the trade offer document to accept it
c) If successful, create a notification for the receiver
d) If successful, return the traded items
e) If any of a) to d) fail, log the error and return `undefined` instead
4. Return `undefined`
In the above steps, you can see some problems with your promise chaining. But aside from that, you can also see that you create and edit documents one-by-one instead of all-at-once ("atomically"). If any of these writes were to fail, your database ends up in an unknown state. As an example, you could have created and accepted a trade, but failed to create the notification.
To atomically write to your database, you need to use a batched write where you bundle a bunch of changes together and then send them off to Firestore. If any of them were to fail, no data is changed in the database.
Next, you store a user's notifications inside of their user document. For a small number of notifications this is fine, but do you need to download all of those notifications if you wanted to pull just an address or phone number like in the example in the above section? I recommend splitting them out into their own document (such as /users/{someUserId}/metadata/notifications), but ideally their own collection (such as /users/{someUserId}/notifications/{someNotificationID}). By placing them in their own collection, you can query them and use QuerySnapshot#docChanges to synchronize changes and use Cloud Firestore triggers to send push notifications.
Refactored Function
1. Get the trade offer document</li>
2. Once the retrieved, do the following depending on the result:
- If failed or empty, return an error
- If successful, do the following:
a) Pull out the sender and receiver's IDs, along with any items in the trade.
b) For each item in the trade, check if any are unavailable and once the check has completed, do the following depending on the result:
- If any item is unavailable, do the following:
a) Decline the trade
b) Return the list of unavailable items
- If all items are available, do the following:
a) Create a new write batch containing:
- Create a document about the trade
- Edit the trade offer document to accept it
- Create a notification for the receiver
b) Commit the write batch to Firestore
c) Once the commit has completed, do the following depending on the result:
- If failed, return an error
- If successful, return the traded items and the trade's ID
Because the steps here depend on each other, this is a good candidate to use async/await syntax.
To see this in action, closely study this:
import * as firebase from "firebase-admin";
// insert here: https://gist.github.com/samthecodingman/aea3bc9481bbab0a7fbc72069940e527
async function firebaseAcceptTradeOffer(tradeOfferID, userData) {
const tradeOfferDocRef = db.collection("tradeOffers").doc(tradeOfferID);
const tradeDoc = await tradeOfferDocRef.get();
const { senderID, receiverID, sendersItems, receiversItems } =
tradeDoc.data();
const itemsArray = sendersItems.concat(receiversItems);
// TODO: Check if this is an accurate assumption
if (sendersItems.length == 0 || receiversItems.length == 0) {
success: false,
message: "One-sided trades are not permitted",
detail: {
sendersItemsIDs: sendersItems.map(({ itemID }) => itemID),
receiversItemsIDs: receiversItems.map(({ itemID }) => itemID),
},
};
const listingsColQuery = db
.collection("listings")
.where("status", "==", "listed");
const uniqueItemIds = Array.from(
itemsArray.reduce(
(set, { itemID }) => set.add(itemID),
new Set()
)
);
const foundIds = {};
await fetchDocumentsWithId(
listingsColQuery,
uniqueItemIds,
(listingDoc) => {
// if here, listingDoc must exist because we used .where("status") above
foundIds[listingDoc.id] = true;
}
);
const unavailableItemIDs = uniqueItemIds
.filter(id => !foundIds[id]);
if (unavailableItems.length > 0) {
// one or more items are unavailable!
await tradeOfferDocRef.update({
status: "declined",
});
return {
success: false,
message: "Some items were unavailable",
detail: {
unavailableItemIDs,
},
};
}
const tradeDocRef = db.collection("trades").doc();
const tradeInstanceID = tradeDocRef.id;
const batch = db.batch();
batch.set(tradeDocRef, {
tradeOfferID,
senderTradeStatus: {
created: true,
sentToSeekio: "current",
inspection: false,
sentToPartner: false,
},
receiverTradeStatus: {
created: true,
sentToSeekio: "current",
inspection: false,
sentToPartner: false,
},
postagePhotos: [],
inspectionPhotos: [],
senderPaid: false,
receiverPaid: false,
senderUploadedProof: false,
receiverUploadedProof: false,
senderID,
receiverID,
messages: [
{
message: `Trade created. A representative, will message this chat shortly with instructions and postage address. If you would like more information about the trading process, head to seekio.io/help. Thank you for using Seekio!`,
sender: "System",
timestamp: firebase.firestore.Timestamp.fromDate(new Date()),
},
],
});
batch.set(
tradeOfferDocRef,
{
status: "accepted",
tradeInstanceID,
},
{ merge: true }
);
const receiverNotificationRef = db
.collection("users")
.doc(senderID)
.collection("notifications")
.doc();
batch.set(receiverNotificationRef, {
from: auth.currentUser.uid,
fromUsername: userData.username,
type: "tradeOfferAccepted",
time: firebase.firestore.Timestamp.fromDate(new Date()),
seen: false,
});
await batch.commit();
return {
success: true,
message: "Trade accepted",
detail: {
tradeID: tradeInstanceID,
senderItems,
receiversItems,
},
};
}
Usage:
try {
const tradeResult = await firebaseAcceptTradeOffer(someTradeId);
} catch (err) {
// if here, one of the following things happened:
// - syntax error
// - database read/write error
// - database rejected batch write
}
In general, when you are returning a promise where it can't be resolved you must await its result. Additionally, you must be returning a value from within a promise then chain, at minimal the last .then() needs to be returning a value, this can also be done within a .finally() method.
Using Get from any firebase resource, realtime, firestore, and storage are all Async processes and must be awaited. in your case, you are missing an await for the return:
var tradeOffer = db.collection("tradeOffers").doc(tradeOfferID);
return tradeOffer
and you don't appear to be returning anything inside your .then() statements, I would suggest a complete rewrite of what you are trying to so you are returning values as they are needed.
So I created an indexed db and then intended to store the data fetched from an api response. But there is an error, I've tried to check whether the data from the response came in or not and the results came in, but it still got an error, and also tried changing the method to put ().
Error
**Uncaught (in promise) DOMException: Failed to execute 'add' on 'IDBObjectStore': Evaluating the object store's key path did not yield a value.**
Index
const playerDetails = ShowPlayerDetails();
const savePlayerButton = document.getElementById("savePlayerButton");
savePlayerButton.addEventListener("click", event => {
playerDetails.then(response => {
dbInsertPlayer(response);
});
});
IndexedDB
const idbPromised = idb.open("player-favourite-list", 1, upgradeDb => {
if(!upgradeDb.objectStoreNames.contains("players")){
upgradeDb.createObjectStore("players", {autoIncrement: true});
}
});
const dbInsertPlayer = playerData => {
return new Promise((resolve, reject) => {
idbPromised.then(db => {
const transaction = db.transaction("players", "readwrite");
console.log(playerData);
return transaction.objectStore("players").put(playerData);
})
.then(transaction => {
if(transaction.complete){
resolve(true);
}
})
.catch(error => {
reject(error);
});
});
}
FetchAPI
function ShowPlayerDetails(){
return new Promise ((resolve, reject) => {
let urlParams = new URLSearchParams(window.location.search);
let idParam = urlParams.get("id");
FetchAPI(`${BASE_URL}players/${idParam}`)
.then(response => {
const {name, nationality, countryOfBirth, dateOfBirth, position} = response;
document.getElementById("player-name").innerHTML = name;
document.getElementById("player-data").innerHTML = `
<ul class="collection">
<li class="collection-item">Name : ${name}</li>
<li class="collection-item">Nationality : ${nationality}</li>
<li class="collection-item">Country Of Birth : ${countryOfBirth}</li>
<li class="collection-item">Date Of Birth : ${dateOfBirth}</li>
<li class="collection-item">Position : ${position}</li>
</ul>`;
resolve(response);
})
.catch(error => reject(error));
})
}
My goal is to enter response data into indexeddb, then display it at another time.
API Response
countryOfBirth: "Senegal"
dateOfBirth: "1992-04-10"
firstName: "Sadio"
id: 3626
lastName: null
lastUpdated: "2020-08-13T03:13:10Z"
name: "Sadio Mané"
nationality: "Senegal"
position: "Attacker"
shirtNumber: null
So I add autoIncrement and keyPath inside the object store, and don't forget to delete the database via the Storage Browser or upgrade the current indexed DB version.
Code
const idbPromised = idb.open("player-favourite-list", 1, upgradeDb => {
if(!upgradeDb.objectStoreNames.contains("players")){
upgradeDb.createObjectStore("players", {keyPath: "Id", autoIncrement: true});
}
});
Exception Refference MDN Indexed DB Documentation