Reusing websocket connection object in Redux Saga - javascript

I have a problem. I am connecting to the server with the help of websocket inside React Redux eventChannel. I have no problems with receivng event from the server but i cant sent event to the server. I understand, that i need to get websocket connection object which i create inside eventChannel to send event from react component for example.
export function* wsWatcher() {
yield takeEvery("LOGIN_SUCCESS", wsConnectionWorker);
}
function* wsConnectionWorker() {
const channel: EventChannel<boolean> = yield call(initWebsocket);
while (true) {
const action: Action<any> = yield take(channel);
yield put(action);
}
}
function initWebsocket(): EventChannel<any> {
return eventChannel((emitter) => {
let id: string | undefined;
let intervalId: number;
const initConnection = () => {
const connectionUrl: string = "ws://localhost:4000"
let ws = new WebSocket(connectionUrl);
ws.onopen = () => {
id && ws.send(JSON.stringify({ type: "GET_USER_ID", id }));
intervalId = window.setInterval(() => {
ws.send(JSON.stringify({ type: "PING" }))
}, 30000)
};
ws.onclose = (e) => {
console.log("Reconnect in 4s");
intervalId && clearInterval(intervalId);
id && setTimeout(initConnection, 4000);
};
};
initConnection();
return () => {
console.log("Socket off");
};
});
}
Thanks in advance

You can create a variable in the module's scope and store connection object in it after first init. Later whenever you need the connection, just use this variable.
This works since modules are cached in nodejs. That is, the code in modules in run only when they're first imported, and later imports use the cached module.
For further reference, read more: Modules Caching
You can also have a function for this, like getConnection function in this code. Whenever you need the websocket connection, call this function and it'll create connection only once.
export function* wsWatcher() {
yield takeEvery("LOGIN_SUCCESS", wsConnectionWorker);
}
function* wsConnectionWorker() {
const channel: EventChannel<boolean> = yield call(getConnection);
while (true) {
const action: Action<any> = yield take(channel);
yield put(action);
}
}
// store the EventChannel after first init
let wsConnection: EventChannel<any> = null;
// create WS connection on first call
// reuse that connection in further calls
function getConnection(): EventChannel<any>{
if(!wsConnection){
wsConnection = initWebsocket();
return wsConnection;
}
function initWebsocket(): EventChannel<any> {
return eventChannel((emitter) => {
let id: string | undefined;
let intervalId: number;
const initConnection = () => {
const connectionUrl: string = "ws://localhost:4000"
let ws = new WebSocket(connectionUrl);
ws.onopen = () => {
id && ws.send(JSON.stringify({ type: "GET_USER_ID", id }));
intervalId = window.setInterval(() => {
ws.send(JSON.stringify({ type: "PING" }))
}, 30000)
};
ws.onclose = (e) => {
console.log("Reconnect in 4s");
intervalId && clearInterval(intervalId);
id && setTimeout(initConnection, 4000);
};
};
initConnection();
return () => {
console.log("Socket off");
};
});
}

Related

WebSocket readyState stuck at 0 after a couple of messages are sent

I am trying to develop a live chat app using web sockets and react, but after I try submitting several messages (around 30) the web socket gets stuck on the CONNECTING state. I have it set up so when it does send a message it disabled the send button to prevent users from spamming messages too fast but I unfortunately still get the same issue.
// id is a uuid() string
const ws = new WebSocket(`ws://localhost:3001/chat/${id}`);
useEffect(() => {
ws.onmessage = function(evt){
try{
const user_id = parseInt(evt.data.split("")[0]);
const message = evt.data.slice(1);
const currentTime = new Date();
const currentUTC = currentTime.toUTCString();
const timestamp = new Date(currentUTC);
setMessages(messages => [...messages, {user_id, message, timestamp}])
} catch(e){
console.log(e);
}
}
ws.onclose = function(evt){
console.log("DISCONNECTED!!")
ws.close();
}
ws.onerror = function(evt){
console.log(evt);
ws.close();
}
}, []);
useEffect(() => {
async function postMessageToAPI() {
const messsageToSend = {
unique_id: id,
message: formData.message,
user_id: user.id,
group_chat_id: room.id
}
// Convert to unviersal time UTC and send it to database
let currentUTC = new Date();
currentUTC.toUTCString();
messsageToSend.timestamp = currentUTC;
await AnonChatApi.sendChatMessage(messsageToSend);
}
if(sendMessage){
ws.onopen = function(){
// add user_id to the start of the message string
const message = `${user.id}` + formData.message;
ws.send(message);
}
postMessageToAPI();
resetFormData();
setTimeout(() => {
setSendMessage(false)
}, 1000);
}
}, [sendMessage]);
const goBackHome = () => {
ws.close();
history.push('/');
}
I can see you're using Hooks, so you must also be using Function Components.
Am I correct in thinking that the code to initialize the websocket
const ws = new WebSocket(`ws://localhost:3001/chat/${id}`);
is at the top of the function?
As a reminder, the function which defines your Function Component is run whenever your component is rendered. Anything that isn't saved in state is lost. This includes your websocket - a new one will be created every render, your async functions may sending data on an old websocket (from a previous render), and React may warn you in the console that you have a memory leak.
useEffect is the proper approach here, but the websocket also needs to be saved in state.
YourFunctionComponent() {
const [ws, setWs] = useState(null);
useEffect(() => {
if (ws == null) {
setWs(new WebSocket(`ws://localhost:3001/chat/${id}`));
}
return () => {
// A function returned from useEffect will
// get called on component unmount.
// Use this function to clean up your connection and
// close your websocket!
// clean up, e.g.
// ws.send('closing due to unmount!');
ws.close();
setWs(null);
}
}, [ws, setWs]);
// Add `ws` as a dependency in the useEffect()s you posted above
useEffect(() => {
ws.onmessage = function(evt){
[--snip--]
}
}, [ws]);
useEffect(() => {
async function postMessageToAPI() {
[--snip--]
}
}, [sendMessage, ws]);
}

Handle multiple calls to async function

I have a imported component, she calls a function every time that user do something (lets say press on button), in the function i must to fetch some data in async order, i would like to run the function calls as async way, the calls to the function will wait until the function is finished and then call the function again.
code example- if i trigger the function 3 times fast:
hadlechange = async(type:string) => {
console.log(1111111)
let storage = await getData("G");
console.log(22222222)
await bla1();
await bla2();
console.log(333333)
await storeData('blabla');
console.log(4444444)
};
render() {
return (
<BlaBla onChange ={this.hadlechange}>)
}
Expected results
11111
22222
3333
4444
1111
2222
3333
4444
1111
2222
3333
4444
What i get
1111
1111
2222
2222
1111
3333
2222
4444
3333
3333
4444
4444
I use JavaScript- React for the client
Thanks to #Proximo i thought about this solution and it's works fine.
Maybe someone else will find it helpful so i share my code :)
constructor() {
this.notHandling = true;
this.saves = [];
}
onChange = async (status: string) => {
this.saves.push(status);
if (this.notHandling) {
this.notHandling = false;
while (saves.length > 0) {
await blabla1(saves.pop());
...
}
this.notHandling = true;
}
};
render() {
return (
<BlaBla onChange ={this.hadlechange}>)
}
Edit
As a help function
export const handleMultiAsyncCalls = func => {
let notHandling = true;
let saves = [];
return async function() {
const args = arguments;
const context = this;
saves.push(args);
if (notHandling) {
notHandling = false;
while (saves.length > 0) {
await func.apply(context, saves.pop());
}
notHandling = true;
}
};
};
In your class you call it this way-
constructor(props) {
super(props);
this.handleMultiCalls = handleMultiAsyncCalls(blablaFunc);
handleChange = (data: string) => {
this.handleMultiCalls(data);
};
What's happening is that multiple changes are happening at once. An easy fix is to set a global flag to prevent the event method while the current action is in-progress.
constructor() {
this.notHandling = true;
}
hadlechange = async(type:string) => {
if (this.notHandling) {
this.notHandling = false;
console.log(1111111)
let storage = await getData("G");
console.log(22222222)
await bla1();
await bla2();
console.log(333333)
await storeData('blabla');
console.log(4444444)
this.notHandling = true;
}
};
Edit: Helper class example
class AsyncQueue {
private queue: Function[];
private processing: boolean;
constructor() {
this.queue = [];
this.processing = false;
}
push = (method:any) => {
this.queue.push(method);
this.process();
}
private process = async () => {
if (!this.processing) {
this.processing = true;
for (let action of this.queue) {
await action();
}
this.processing = false;
}
}
}
export const asyncQueue = new AsyncQueue;
Using Helper
// import example...
import { asyncQueue } from '~/common/helpers';
// Async test method
const tester = (message: string, seconds: number = 1) => {
return new Promise((resolve) => {
console.log('start ' + message);
setTimeout(() => {
console.log(message);
resolve(message);
}, seconds * 1000);
});
};
// Method calls
asyncQueue.push(() => tester('FIRST'));
setTimeout(() => asyncQueue.push(() => tester('SECOND', 2)), 50);
asyncQueue.push(() => tester('THIRD', 4));

React useState hook not consistently updating

I started integrating websockets into an existing React/Django app following along with this example (accompanying repo here). In that repo, the websocket interface is in websockets.js, and is implemented in containers/Chat.js.
I can get that code working correctly as-is.
I then started re-writing my implementation to use Hooks, and hit a little wall. The data flows through the socket correctly, arrives in the handler of each client correctly, and within the handler can read the correct state. Within that handler, I'm calling my useState function to update state with the incoming data.
Originally I had a problem of my single useState function within addMessage() inconsistently firing (1 in 10 times?). I split my one useState hook into two (one for current message, one for all messages). Now in addMessage() upon receiving data from the server, my setAllMessages hook will only update the client where I type the message in - no other clients. All clients receive/can log the data correctly, they just don't run the setAllMessages function.
If I push to an empty array outside the function, it works as expected. So it seems like a problem in the function update cycle, but I haven't been able to track it down.
Here's my version of websocket.js:
class WebSocketService {
static instance = null;
static getInstance() {
if (!WebSocketService.instance) {
WebSocketService.instance = new WebSocketService();
}
return WebSocketService.instance;
}
constructor() {
this.socketRef = null;
this.callbacks = {};
}
disconnect() {
this.socketRef.close();
}
connect(chatUrl) {
const path = `${URLS.SOCKET.BASE}${URLS.SOCKET.TEST}`;
this.socketRef = new WebSocket(path);
this.socketRef.onopen = () => {
console.log('WebSocket open');
};
this.socketRef.onmessage = e => {
this.socketNewMessage(e.data);
};
this.socketRef.onerror = e => {
console.log(e.message);
};
this.socketRef.onclose = () => {
this.connect();
};
}
socketNewMessage(data) {
const parsedData = JSON.parse(data);
const { command } = parsedData;
if (Object.keys(this.callbacks).length === 0) {
return;
}
Object.keys(SOCKET_COMMANDS).forEach(clientCommand => {
if (command === SOCKET_COMMANDS[clientCommand]) {
this.callbacks[command](parsedData.presentation);
}
});
}
backend_receive_data_then_post_new(message) {
this.sendMessage({
command_for_backend: 'backend_receive_data_then_post_new',
message: message.content,
from: message.from,
});
}
sendMessage(data) {
try {
this.socketRef.send(JSON.stringify({ ...data }));
} catch (err) {
console.log(err.message);
}
}
addCallbacks(allCallbacks) {
Object.keys(SOCKET_COMMANDS).forEach(command => {
this.callbacks[SOCKET_COMMANDS[command]] = allCallbacks;
});
}
state() {
return this.socketRef.readyState;
}
}
const WebSocketInstance = WebSocketService.getInstance();
export default WebSocketInstance;
And here's my version of Chat.js
export function Chat() {
const [allMessages, setAllMessages] = useState([]);
const [currMessage, setCurrMessage] = useState('');
function waitForSocketConnection(callback) {
setTimeout(() => {
if (WebSocketInstance.state() === 1) {
callback();
} else {
waitForSocketConnection(callback);
}
}, 100);
}
waitForSocketConnection(() => {
const allCallbacks = [addMessage];
allCallbacks.forEach(callback => {
WebSocketInstance.addCallbacks(callback);
});
});
/*
* This is the problem area
* `incoming` shows the correct data, and I have access to all state
* But `setAllMessages` only updates on the client I type the message into
*/
const addMessage = (incoming) => {
setAllMessages([incoming]);
};
// update with value from input
const messageChangeHandler = e => {
setCurrMessage(e.target.value);
};
// Send data to socket interface, then to server
const sendMessageHandler = e => {
e.preventDefault();
const messageObject = {
from: 'user',
content: currMessage,
};
setCurrMessage('');
WebSocketInstance.backend_receive_data_then_post_new(messageObject);
};
return (
<div>
// rendering stuff here
</div>
);
}
There is no need to rewrite everything into functional components with hooks.
You should decompose it functionally - main (parent, class/FC) for initialization and providing [data and] methods (as props) to 2 functional childrens/components responsible for rendering list and input (new message).
If you still need it ... useEffect is a key ... as all code is run on every render in functional components ... including function definitions, redefinitions, new refs, duplications in callbacks array etc.
You can try to move all once defined functions into useEffect
useEffect(() => {
const waitForSocketConnection = (callback) => {
...
}
const addMessage = (incoming) => {
setAllMessages([incoming]);
};
waitForSocketConnection(() => {
...
}
}, [] ); // <<< RUN ONCE

NodeJS script not actioning with async/await

Cannot figure out why the below script won't run. It is likely the script is not going to do what I want but using
node ./contentful/contentful-assets.js
in the terminal, it does nothing - No errors, nothing logged for me to even start debugging. However, if I remove async it will attempt the script and shoot back an error.
./contentful/contentful-assets.js
const contentful = require('contentful-management');
const iterator = require('make-iterator');
const assets = require('./assetObject.js');
async resolve => {
console.log('Creating Contentful client');
const client = contentful.createClient({
accessToken: 'token',
logHandler: (level, data) => console.log(`${level} | ${data}`)
});
const iterableAssets = iterator(assets);
const space = await client.getSpace('space');
const environment = await space.getEnvironment('enviroment');
const cmsAssets = [];
const assetProcessingTimes = [];
const inProcess = new Map();
let processedAssetsCounter = 0;
const createAndPublishSingleAsset = async ({ asset, done, index }) => {
if (done) {
if (inProcess.size > 0) return false;
return resolve(cmsAssets);
}
const start = Date.now();
const id = '' + start + Math.round(Math.random() * 100);
inProcess.set(id, true);
let cmsAsset;
try {
cmsAsset = await environment.createAssetWithId(asset.postId, {
fields: {
title: {
'en-US': asset.title
},
description: {
'en-US': asset.description
},
file: {
'en-US': {
contentType: 'image/jpg',
fileName: asset.filename,
upload: asset.link
}
}
}
});
} catch (e) {
console.log(`Asset "${asset.title}" failed to create, retrying...`);
createAndPublishSingleAsset({
asset,
done,
index
});
}
try {
const processedCMSAsset = await cmsAsset.processForAllLocales();
const publishedCMSAsset = await processedCMSAsset.publish();
cmsAssets.push(publishedCMSAsset);
assetProcessingTimes.push((Date.now() - start) / 1000);
inProcess.clear(id);
const eta = Math.floor(
assetProcessingTimes.reduce((a, b) => a + b, 0) /
assetProcessingTimes.length *
(assets.length - index) /
60
);
processedAssetsCounter += 1;
console.log(
`Processed asset ${processedAssetsCounter}/${assets.length} - eta: ${eta}m`
);
createAndPublishSingleAsset(iterableAssets.next());
} catch (e) {
console.log(`Asset "${asset.title}" failed to process, retrying...`);
await cmsAsset.delete();
createAndPublishSingleAsset({
asset,
done,
index
});
}
};
console.log('Starting to create assets');
createAndPublishSingleAsset(iterableAssets.next());
createAndPublishSingleAsset(iterableAssets.next());
createAndPublishSingleAsset(iterableAssets.next());
};
assetObject.js
[
{
link: 'https://example.com/example1.jpg',
title: 'Example 1',
description: 'Description of example 1',
postId: '1234567890',
filename: 'example1.jpeg'
}, ... // Many more
]
What have I missed here?
I fear that you are not calling the function, could you try, the following?
const contentful = require('contentful-management');
const iterator = require('make-iterator');
const assets = require('./assetObject.js');
const doWork = async resolve => {
console.log('Creating Contentful client');
...
}
doWork();
You are just declaring a function that is async and does all of the code defined, but you are not actually calling it.
In this code snippet you are declaring a function, but never invoking it:
//declaring an async function, with "resolve" as the argument
async resolve => {
//function definition
}
In order to be able to later reference the function to invoke you can assign it to const/let/etc.:
const createAssets = async resolve => { }
//now, invoke
createAssets()

How can I update more than 500 docs in Firestore using Batch?

I'm trying to update a field timestamp with the Firestore admin timestamp in a collection with more than 500 docs.
const batch = db.batch();
const serverTimestamp = admin.firestore.FieldValue.serverTimestamp();
db
.collection('My Collection')
.get()
.then((docs) => {
serverTimestamp,
}, {
merge: true,
})
.then(() => res.send('All docs updated'))
.catch(console.error);
This throws an error
{ Error: 3 INVALID_ARGUMENT: cannot write more than 500 entities in a single call
at Object.exports.createStatusError (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\common.js:87:15)
at Object.onReceiveStatus (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:1188:28)
at InterceptingListener._callNext (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:564:42)
at InterceptingListener.onReceiveStatus (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:614:8)
at callback (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:841:24)
code: 3,
metadata: Metadata { _internal_repr: {} },
details: 'cannot write more than 500 entities in a single call' }
Is there a way that I can write a recursive method which creates a batch object updating a batch of 500 docs one by one until all the docs are updated.
From the docs I know that delete operation is possible with the recursive approach as mentioned here:
https://firebase.google.com/docs/firestore/manage-data/delete-data#collections
But, for updating, I'm not sure how to end the execution since the docs are not being deleted.
I also ran into the problem to update more than 500 documents inside a Firestore collection. And i would like to share how i solved this problem.
I use cloud functions to update my collection inside Firestore but this should also work on client side code.
The solution counts every operation which is made to the batch and after the limit is reached a new batch is created and pushed to the batchArray.
After all updates are completed the code loops through the batchArray and commits every batch which is inside the array.
It is important to count every operation set(), update(), delete() which is made to the batch because they all count to the 500 operation limit.
const documentSnapshotArray = await firestore.collection('my-collection').get();
const batchArray = [];
batchArray.push(firestore.batch());
let operationCounter = 0;
let batchIndex = 0;
documentSnapshotArray.forEach(documentSnapshot => {
const documentData = documentSnapshot.data();
// update document data here...
batchArray[batchIndex].update(documentSnapshot.ref, documentData);
operationCounter++;
if (operationCounter === 499) {
batchArray.push(firestore.batch());
batchIndex++;
operationCounter = 0;
}
});
batchArray.forEach(async batch => await batch.commit());
return;
I liked this simple solution:
const users = await db.collection('users').get()
const batches = _.chunk(users.docs, 500).map(userDocs => {
const batch = db.batch()
userDocs.forEach(doc => {
batch.set(doc.ref, { field: 'myNewValue' }, { merge: true })
})
return batch.commit()
})
await Promise.all(batches)
Just remember to add import * as _ from "lodash" at the top. Based on this answer.
You can use default BulkWriter. This method used 500/50/5 rule.
Example:
let bulkWriter = firestore.bulkWriter();
bulkWriter.create(documentRef, {foo: 'bar'});
bulkWriter.update(documentRef2, {foo: 'bar'});
bulkWriter.delete(documentRef3);
await close().then(() => {
console.log('Executed all writes');
});
As mentioned above, #Sebastian's answer is good and I upvoted that too. Although faced an issue while updating 25000+ documents in one go.
The tweak to logic is as below.
console.log(`Updating documents...`);
let collectionRef = db.collection('cities');
try {
let batch = db.batch();
const documentSnapshotArray = await collectionRef.get();
const records = documentSnapshotArray.docs;
const index = documentSnapshotArray.size;
console.log(`TOTAL SIZE=====${index}`);
for (let i=0; i < index; i++) {
const docRef = records[i].ref;
// YOUR UPDATES
batch.update(docRef, {isDeleted: false});
if ((i + 1) % 499 === 0) {
await batch.commit();
batch = db.batch();
}
}
// For committing final batch
if (!(index % 499) == 0) {
await batch.commit();
}
console.log('write completed');
} catch (error) {
console.error(`updateWorkers() errored out : ${error.stack}`);
reject(error);
}
Explanations given on previous comments already explain the issue.
I'm sharing the final code that I built and worked for me, since I needed something that worked in a more decoupled manner, instead of the way that most of the solutions presented above do.
import { FireDb } from "#services/firebase"; // = firebase.firestore();
type TDocRef = FirebaseFirestore.DocumentReference;
type TDocData = FirebaseFirestore.DocumentData;
let fireBatches = [FireDb.batch()];
let batchSizes = [0];
let batchIdxToUse = 0;
export default class FirebaseUtil {
static addBatchOperation(
operation: "create",
ref: TDocRef,
data: TDocData
): void;
static addBatchOperation(
operation: "update",
ref: TDocRef,
data: TDocData,
precondition?: FirebaseFirestore.Precondition
): void;
static addBatchOperation(
operation: "set",
ref: TDocRef,
data: TDocData,
setOpts?: FirebaseFirestore.SetOptions
): void;
static addBatchOperation(
operation: "create" | "update" | "set",
ref: TDocRef,
data: TDocData,
opts?: FirebaseFirestore.Precondition | FirebaseFirestore.SetOptions
): void {
// Lines below make sure we stay below the limit of 500 writes per
// batch
if (batchSizes[batchIdxToUse] === 500) {
fireBatches.push(FireDb.batch());
batchSizes.push(0);
batchIdxToUse++;
}
batchSizes[batchIdxToUse]++;
const batchArgs: [TDocRef, TDocData] = [ref, data];
if (opts) batchArgs.push(opts);
switch (operation) {
// Specific case for "set" is required because of some weird TS
// glitch that doesn't allow me to use the arg "operation" to
// call the function
case "set":
fireBatches[batchIdxToUse].set(...batchArgs);
break;
default:
fireBatches[batchIdxToUse][operation](...batchArgs);
break;
}
}
public static async runBatchOperations() {
// The lines below clear the globally available batches so we
// don't run them twice if we call this function more than once
const currentBatches = [...fireBatches];
fireBatches = [FireDb.batch()];
batchSizes = [0];
batchIdxToUse = 0;
await Promise.all(currentBatches.map((batch) => batch.commit()));
}
}
Based on all the above answers, I put together the following pieces of code that one can put into a module in JavaScript back-end and front-end to easily use Firestore batch writes, without worrying about the 500 writes limit.
Back-end (Node.js)
// The Firebase Admin SDK to access Firestore.
const admin = require("firebase-admin");
admin.initializeApp();
// Firestore does not accept more than 500 writes in a transaction or batch write.
const MAX_TRANSACTION_WRITES = 499;
const isFirestoreDeadlineError = (err) => {
console.log({ err });
const errString = err.toString();
return (
errString.includes("Error: 13 INTERNAL: Received RST_STREAM") ||
errString.includes("Error: 4 DEADLINE_EXCEEDED: Deadline exceeded")
);
};
const db = admin.firestore();
// How many transactions/batchWrites out of 500 so far.
// I wrote the following functions to easily use batchWrites wthout worrying about the 500 limit.
let writeCounts = 0;
let batchIndex = 0;
let batchArray = [db.batch()];
// Commit and reset batchWrites and the counter.
const makeCommitBatch = async () => {
console.log("makeCommitBatch");
await Promise.all(batchArray.map((bch) => bch.commit()));
};
// Commit the batchWrite; if you got a Firestore Deadline Error try again every 4 seconds until it gets resolved.
const commitBatch = async () => {
try {
await makeCommitBatch();
} catch (err) {
console.log({ err });
if (isFirestoreDeadlineError(err)) {
const theInterval = setInterval(async () => {
try {
await makeCommitBatch();
clearInterval(theInterval);
} catch (err) {
console.log({ err });
if (!isFirestoreDeadlineError(err)) {
clearInterval(theInterval);
throw err;
}
}
}, 4000);
}
}
};
// If the batchWrite exeeds 499 possible writes, commit and rest the batch object and the counter.
const checkRestartBatchWriteCounts = () => {
writeCounts += 1;
if (writeCounts >= MAX_TRANSACTION_WRITES) {
batchIndex++;
batchArray.push(db.batch());
writeCounts = 0;
}
};
const batchSet = (docRef, docData) => {
batchArray[batchIndex].set(docRef, docData);
checkRestartBatchWriteCounts();
};
const batchUpdate = (docRef, docData) => {
batchArray[batchIndex].update(docRef, docData);
checkRestartBatchWriteCounts();
};
const batchDelete = (docRef) => {
batchArray[batchIndex].delete(docRef);
checkRestartBatchWriteCounts();
};
module.exports = {
admin,
db,
MAX_TRANSACTION_WRITES,
checkRestartBatchWriteCounts,
commitBatch,
isFirestoreDeadlineError,
batchSet,
batchUpdate,
batchDelete,
};
Front-end
// Firestore does not accept more than 500 writes in a transaction or batch write.
const MAX_TRANSACTION_WRITES = 499;
const isFirestoreDeadlineError = (err) => {
return (
err.message.includes("DEADLINE_EXCEEDED") ||
err.message.includes("Received RST_STREAM")
);
};
class Firebase {
constructor(fireConfig, instanceName) {
let app = fbApp;
if (instanceName) {
app = app.initializeApp(fireConfig, instanceName);
} else {
app.initializeApp(fireConfig);
}
this.name = app.name;
this.db = app.firestore();
this.firestore = app.firestore;
// How many transactions/batchWrites out of 500 so far.
// I wrote the following functions to easily use batchWrites wthout worrying about the 500 limit.
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
}
async makeCommitBatch() {
console.log("makeCommitBatch");
if (!this.isCommitting) {
this.isCommitting = true;
await this.batch.commit();
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.isCommitting = true;
await this.batch.commit();
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async commitBatch() {
try {
await this.makeCommitBatch();
} catch (err) {
console.log({ err });
if (isFirestoreDeadlineError(err)) {
const theInterval = setInterval(async () => {
try {
await this.makeCommitBatch();
clearInterval(theInterval);
} catch (err) {
console.log({ err });
if (!isFirestoreDeadlineError(err)) {
clearInterval(theInterval);
throw err;
}
}
}, 4000);
}
}
}
async checkRestartBatchWriteCounts() {
this.writeCounts += 1;
if (this.writeCounts >= MAX_TRANSACTION_WRITES) {
await this.commitBatch();
}
}
async batchSet(docRef, docData) {
if (!this.isCommitting) {
this.batch.set(docRef, docData);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.set(docRef, docData);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async batchUpdate(docRef, docData) {
if (!this.isCommitting) {
this.batch.update(docRef, docData);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.update(docRef, docData);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async batchDelete(docRef) {
if (!this.isCommitting) {
this.batch.delete(docRef);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.delete(docRef);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
}
No citations or documentation, this code i invented by myself and for me it worked and looks clean, and simple for read and usage. If some one like it, then can use it too.
Better make autotest becose code use private var _ops wich can be changed after packages upgrade. Forexample in old versions its can be _mutations
async function commitBatch(batch) {
const MAX_OPERATIONS_PER_COMMIT = 500;
while (batch._ops.length > MAX_OPERATIONS_PER_COMMIT) {
const batchPart = admin.firestore().batch();
batchPart._ops = batch._ops.splice(0, MAX_OPERATIONS_PER_COMMIT - 1);
await batchPart.commit();
}
await batch.commit();
}
Usage:
const batch = admin.firestore().batch();
batch.delete(someRef);
batch.update(someRef);
...
await commitBatch(batch);
Simple solution
Just fire twice ?
my array is "resultsFinal"
I fire batch once with a limit of 490 , and second with a limit of the lenght of the array ( results.lenght)
Works fine for me :)
How you check it ?
You go to firebase and delete your collection , firebase say you have delete XXX docs , same as the lenght of your array ? Ok so you are good to go
async function quickstart(results) {
// we get results in parameter for get the data inside quickstart function
const resultsFinal = results;
// console.log(resultsFinal.length);
let batch = firestore.batch();
// limit of firebase is 500 requests per transaction/batch/send
for (i = 0; i < 490; i++) {
const doc = firestore.collection('testMore490').doc();
const object = resultsFinal[i];
batch.set(doc, object);
}
await batch.commit();
// const batchTwo = firestore.batch();
batch = firestore.batch();
for (i = 491; i < 776; i++) {
const objectPartTwo = resultsFinal[i];
const doc = firestore.collection('testMore490').doc();
batch.set(doc, objectPartTwo);
}
await batch.commit();
}

Categories