We Keep Coding

SQS messages are coming back to DLQ?

I have written the following script to move messages from DLQ to MainQ.
'use strict'
import { readFileSync } from "fs";
import {
DeleteMessageCommand,
SQSClient,
GetQueueUrlCommand,
SendMessageCommand,
ReceiveMessageCommand,
GetQueueAttributesCommand
} from '#aws-sdk/client-sqs';
const REGION = 'us-east-1';
const sqs = new SQSClient({ region: REGION });
async function getMessageCount(queueUrl) {
const params = { AttributeNames: ['ApproximateNumberOfMessages'], QueueUrl: queueUrl };
try {
const data = await sqs.send(new GetQueueAttributesCommand(params));
// console.log(data);
return data.Attributes.ApproximateNumberOfMessages;
} catch (err) {
console.error(err);
}
}
async function reprocessMessages(dlqQueue, mainQueue) {
try {
const response1 = await sqs.send(new GetQueueUrlCommand({ QueueName: dlqQueue }));
const response2 = await sqs.send(new GetQueueUrlCommand({ QueueName: mainQueue }));
const dlqQueueUrl = response1.QueueUrl;
const mainQueueUrl = response2.QueueUrl;
const messageCount = await getMessageCount(dlqQueueUrl);
console.log(`Message count ${messageCount}`);
const visibiltyTimeout = Math.ceil(messageCount / 100) * 2;
let count = 0;
while (count <= 50) {
count += await moveMessage(mainQueueUrl, dlqQueueUrl, 5);
}
console.log(`Moved ${count} messages`);
return "Completed";
} catch (err) {
console.error(err);
}
}
async function moveMessage(mainQueueUrl, dlqQueueUrl, visibiltyTimeout) {
try {
const receiveMessageParams = {
MaxNumberOfMessages: 9,
MessageAttributeNames: ['All'],
QueueUrl: dlqQueueUrl,
VisibilityTimeout: visibiltyTimeout,
WaitTimeSeconds: 1
};
const receiveData = await sqs.send(new ReceiveMessageCommand(receiveMessageParams));
// console.log(receiveData);
if (!receiveData.Messages) {
// console.log("finished");
return 0;
}
const messages = [];
receiveData.Messages.forEach(msg => {
messages.push({ id: msg.MessageId, msgAttributes: msg.MessageAttributes, body: msg.Body, receiptHandle: msg.ReceiptHandle });
});
const sendMsg = async ({ id, msgAttributes, body, _ }) => {
const sendMessageParams = {
MessageId: id,
MessageAttributes: msgAttributes,
MessageBody: body,
QueueUrl: mainQueueUrl
};
const sentData = await sqs.send(new SendMessageCommand(sendMessageParams));
// console.log("Success, message sent. MessageID: ", sentData.MessageId);
return 'Success';
};
const deleteMsg = async ({ id, receiptHandle }) => {
const deleteMessageParams = {
MessageId: id,
QueueUrl: dlqQueueUrl,
ReceiptHandle: receiptHandle
};
const deleteData = await sqs.send(new DeleteMessageCommand(deleteMessageParams));
// console.log("Message deleted", deleteData);
return 'Deleted';
};
const sent = await Promise.all(messages.map(sendMsg));
console.log(sent);
const deleted = await Promise.all(messages.map(deleteMsg));
console.log(deleted);
console.log(sent.length);
return sent.length;
// return 1;
} catch (err) {
console.log(err);
}
}
function main() {
const queues = readFileSync('queues.txt', 'utf8').split('\n');
queues.map(async elem => {
const [dlqQueue, mainQueue] = elem.split(' ');
console.log(`Moving messages from ${dlqQueue} to ${mainQueue}`);
const response = await reprocessMessages(dlqQueue, mainQueue);
console.log(response);
});
}
main()
This script moves the message from DLQ to MainQ and shows the new count of the DLQ decreased by the number of messages. However, when those messages in MainQ fail they again come back to DLQ thus keeping the count same as before reprocessing. But, after a while I see that the messages in DLQ increase by the same amount as the number of messages processed.
For instance for the below case I moved 54 messages and the screenshots for the queue counts are attached after each stage.

Firebase cloud function uploading data to the cloud firebase deletes the data without any reason

Hello so a little bit of background.
I created a function in JS that fetches some data(multiple objects that can change in number every time I fetch the data ad example : fetch1: I get 3 obj | fetch2: I get 1 obj, I CAN NOT GET 0 obj ) from a website every minute, processes that data, and then sends it to the cloud firestore. The problem is that after the upload of the data, the doc and the collection get deleted without any reason at all and I don't understand what is happening.
I tried to search online but I could not find a fix.
Can anyone give me a hint of what is happening ?
This is the function in JS
import functions = require("firebase-functions");
import admin = require("firebase-admin");
admin.initializeApp();
const database = admin.firestore();
import https = require("https");
let finalData: { [x: string]: any; }[] = [];
const page = 1;
const fiat = "";
const tradeType = "";
const asset = "";
const payTypes = [""];
const baseObj = {
page,
rows: ,
publisherType: ,
asset,
tradeType,
fiat,
payTypes,
};
const stringData = JSON.stringify(baseObj);
const options = {
hostname: "",
port: ,
path: "",
method: "",
headers: {
"Content-Type": "application/json",
"Content-Length": stringData.length,
},
};
const req = https.request(options, (res: any) => {
finalData = [];
let output = "";
res.on("data", (d: string) => {
output += d;
});
res.on("end", () => {
try {
const jsonOuput = JSON.parse(output);
const allData = jsonOuput["data"];
for (let i = 0; i < allData.length; i++) {
let payTypesz = "";
for (let y = 0; y < allData[i]["adv"]["tradeMethods"].length; y++) {
payTypesz += allData[i]["adv"]["tradeMethods"][y]["payType"];
if (y < allData[i]["adv"]["tradeMethods"].length - 1) {
payTypesz += ", ";
}
}
const obj = {
tradeType: allData[i]["adv"]["tradeType"],
asset: allData[i]["adv"]["asset"],
fiatUnit: allData[i]["adv"]["fiatUnit"],
price: allData[i]["adv"]["price"],
surplusAmount: allData[i]["adv"]["surplusAmount"],
maxSingleTransAmount: allData[i]["adv"]["maxSingleTransAmount"],
minSingleTransAmount: allData[i]["adv"]["minSingleTransAmount"],
nickName: allData[i]["advertiser"]["nickName"],
monthOrderCount: allData[i]["advertiser"]["monthOrderCount"],
monthFinishRate: allData[i]["advertiser"]["monthFinishRate"],
advConfirmTime: allData[i]["advertiser"]["advConfirmTime"],
payTypes: payTypesz,
position: 0,
};
finalData.push(obj);
}
console.log(finalData);
} catch (e) {
console.log(e);
}
});
});
exports.scheduledFunction = functions.pubsub
.schedule("* * * * *")
.onRun((context: any) => {
req.write(stringData);
req.end();
for (let i = 0; i < finalData.length; i++) {
database.doc("/$i")
.set({
"tradeType": finalData[i]["tradeType"],
"asset": finalData[i]["asset"],
"fiatUnit": finalData[i]["fiatUnit"],
"price": finalData[i]["price"],
"surplusAmount": finalData[i]["surplusAmount"],
"maxSingleTransAmount": finalData[i]["maxSingleTransAmount"],
"minSingleTransAmount": finalData[i]["minSingleTransAmount"],
"nickName": finalData[i]["nickName"],
"monthOrderCount": finalData[i]["monthOrderCount"],
"monthFinishRate": finalData[i]["monthFinishRate"],
"advConfirmTime": finalData[i]["advConfirmTime"],
"payTypes": finalData[i]["payTypes"],
"position": finalData[i]["position"],
});
}
return console.log("Succes Upload of the data ");
});
// # sourceMappingURL=index.js.map
This is the setup of the DB.

By default https for Node.js does not return Promises, therefore it can be cumbersome to correctly manage the life cycle of your Cloud Function.
I would suggest you use the axios library and refactor your code as follows, using a batched write to write to Firestore:
exports.scheduledFunction = functions.pubsub
.schedule("* * * * *")
.onRun(async (context: any) => { // <=== See async keyword here
try {
const httpCallResponse = await axios.get(...); // I let you adapt the code, including the URL to call, according to the axios doc
const finalData = ... // Again, it's up to you to work out the value based on httpCallResponse
const batch = database.batch();
for (let i = 0; i < finalData.length; i++) {
batch.set(database.doc(i.toString(10)),
{
"tradeType": finalData[i]["tradeType"],
"asset": finalData[i]["asset"],
"fiatUnit": finalData[i]["fiatUnit"],
"price": finalData[i]["price"],
"surplusAmount": finalData[i]["surplusAmount"],
"maxSingleTransAmount": finalData[i]["maxSingleTransAmount"],
"minSingleTransAmount": finalData[i]["minSingleTransAmount"],
"nickName": finalData[i]["nickName"],
"monthOrderCount": finalData[i]["monthOrderCount"],
"monthFinishRate": finalData[i]["monthFinishRate"],
"advConfirmTime": finalData[i]["advConfirmTime"],
"payTypes": finalData[i]["payTypes"],
"position": finalData[i]["position"],
});
}
await batch.commit();
console.log("Succes Upload of the data ");
return null;
} catch (error) {
console.log(error);
return true;
}
});
Extra Note: Note that with your code you create monotonically increasing IDs and that could be a problem.

Sending res to client before finishing async processes in Node.js

I have the following code that I can´t figure out how to make this entire process finish before sending a response back to the client. It has a for loop and make the requests to 2 external functions.
The problem that I can see in the console is the response which goes way faster than the process takes to complete the processing. It makes the client believe everything is fine, but, actually some error might has happened. I´ve tried to await, I´ve tried reading other posts, I´ve tried making this to return a promise and return a resolve, I´ve tried an index to check the array length... If someone could help me out, I would appreciate that. Thanks in advance.
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryMaxId)
.then(resultqueryMaxId => {
if(resultqueryMaxId.length){
maxId = resultqueryMaxId[0]['maxId'];
}else{
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database.query(queryAllNodes)
.then( async resultqueryAllNodes => {
for (let index = 0; index < data.children.length; index++) {
const element = data.children[index];
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
database.query(query)
.then( async result => {
if (result.length) { // Update char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else { // Add char in productchar1 shouldn´t get in here
console.log("Erro em: updateProdCharTree > addProductProductChar1");
}
})
.catch(err => {
console.log("Erro em query");
contErros = 1;
})
}else{ // Add new char in productchar1
maxId++;
var params = {element: element, productLine: productLine, bu: bu, index: index, maxId: maxId};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
}
})
.catch(err => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
})
})
.catch(err => {
console.log("Erro em queryMaxId");
contErros = 1;
});
if (contAcertos == data.children.length) {
resolve("result"); // ***RES ATTEMPT***
}
})
}
Beginner here. Please be pacient. You were once either.

For starters, you can refactor your code to something like below so that response is returned after all async functions within loop are executed:
var updateProdCharTree = async (req, res, next) => {
return new Promise((resolve, reject) => {
var data = req.body.raw.data;
var productLine = req.body.raw.productLine;
var bu = req.body.raw.bu;
let contErros = null;
let contAcertos = 0;
var maxId = 0;
let queryMaxId = `SELECT max(ProductCharFatherId) as maxId FROM maxiplastmatriz.productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryMaxId)
.then((resultqueryMaxId) => {
if (resultqueryMaxId.length) {
maxId = resultqueryMaxId[0]["maxId"];
} else {
maxId = 0;
}
let queryAllNodes = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine}`;
database
.query(queryAllNodes)
.then(async (resultqueryAllNodes) => {
return Promise.all(
data.chilren.map(async (element) => {
if (data.children[index].dbId != undefined) {
let query = `SELECT Id FROM productchar1 WHERE bu=${bu} and prodline=${productLine} and Id=${data.children[index].dbId} and ProductCharFatherId=${data.children[index].id}`;
let result = await database.query(query);
if (result.length) {
// Update char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
};
waitResUp = await updateProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
contAcertos++;
} else {
// Add char in productchar1 shouldn´t get in here
console.log(
"Erro em: updateProdCharTree > addProductProductChar1"
);
}
} else {
// Add new char in productchar1
maxId++;
var params = {
element: element,
productLine: productLine,
bu: bu,
index: index,
maxId: maxId,
};
waitRes = await addProductProductChar1(params, req, res); //***CALL EXTERNAL METHOD 2***
console.log("waitRes", waitRes.insertId);
contAcertos++;
}
})
);
})
.then(() => {
// This will be called after all queries are executed
resolve("result");
})
.catch((err) => {
console.log("Erro em queryAllNodes", err);
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
})
.catch((err) => {
console.log("Erro em queryMaxId");
contErros = 1;
// NOTE: Ideally we should be rejecting promise in case of error
});
});
};
As #Jeremy Thille suggested this code can be improvised further to only use async-await instead of Promises with then callback so that it is more cleaner and easy to understand.

Firebase Cloud Functions: Promise not resolving

I am having some issues with my a particular call in my cloud function that doesn't seem to be resolving correctly.
This is the code that doesn't want to resolve:
console.log('Getting Search Patterns');
let searchPatterns: FirebaseFirestore.QuerySnapshot;
try {
searchPatterns = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
} catch (error) {
console.error(error);
}
console.log(`Search Patterns Received: ${searchPatterns}`);
LOG:
As you can see in the log, my function runs up until the console log before the try block, then stops until the function times out. I'm not sure what it is that is causing this issue.
EDIT: I have reformatted my code by separating out each of the different parts in my cloud function into separate functions that I can call; the resulting getSearchTerms() function is as follows:
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
This still stops at the same point in the function execution, the full function is here, this has been updated to the latest version:
import * as functions from 'firebase-functions';
import * as admin from 'firebase-admin';
import * as path from 'path';
import * as WordExtractor from 'word-extractor';
import * as textract from 'textract';
import suffixArray from './suffixArray';
// interface Location {
// lid: string;
// location_name: string;
// location_type: string;
// sentimental_value: number;
// }
// interface Context {
// lid: string;
// context_string: string;
// fid: string;
// }
export const processFile = functions.storage.object().onFinalize(async file => {
const serviceAccount = require(__dirname + '/../config/serviceAccount.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: 'https://fyp-alex.firebaseio.com',
});
const firestore = admin.firestore();
const fileBucket: string = file.bucket;
const filePath: string = file.name;
const fileDet: string = path.basename(filePath);
const fileNameSplit: string[] = fileDet.split('.');
const fileExt: string = fileNameSplit.pop();
const fileName: string = fileNameSplit.join('.');
const bucket = admin.storage().bucket(fileBucket);
const fileRef = bucket.file(filePath);
const _path: string = `/tmp/${fileName}.${fileExt}`;
console.log(`File path ${filePath}`);
console.log('Getting Download URL');
try {
console.log(`Downloading to: ${_path}`);
await fileRef.download({ destination: _path });
console.log('File Saved');
console.log(`Getting Details: ${_path}`);
const text: string = await getText(_path, fileExt);
console.log(`Processing: ${fileName}`);
console.log('Creating Suffix Array');
const suffix_array = suffixArray(text);
console.log(`Suffix Array Created: ${suffix_array}`);
console.log('Getting Search Patterns');
const searchTerms: FirebaseFirestore.DocumentData[] = await getSearchTerms();
console.log('Search Patterns Received');
const promises = [];
const allContexts: Object[] = [];
for (const searchDoc of searchTerms) {
const searchTerm = searchDoc.location_name.toLowerCase();
console.log(searchTerm);
const matchedIndexes = search(text, searchTerm, suffix_array);
const contexts = createContexts(matchedIndexes, searchDoc, text, fileName);
allContexts.concat(contexts);
}
for (const context of allContexts) {
const p = admin
.firestore()
.collection('FYP_CONTEXTS')
.add(context);
promises.push(p);
}
await Promise.all(promises);
const data = {
processed: 1,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
} catch (e) {
console.error(e);
const data = {
processed: 2,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
}
});
async function getText(_path: string, fileExt: string) {
let text: string = '';
switch (fileExt) {
case 'docx':
case 'doc':
const extractor = new WordExtractor();
const extracted = await extractor.extract(_path);
text = extracted.getBody();
break;
case 'pdf':
break;
case 'txt':
textract.fromFileWithPath(_path, function(extractedError: any, string: string) {
if (extractedError) {
console.error(extractedError);
}
if (string !== null) {
text = string;
}
});
break;
default:
console.log('Unsupported File Type');
}
return text;
}
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
function createContexts(
matchedIndexes: number[],
searchDoc: FirebaseFirestore.DocumentData,
text: string,
fileName: string
) {
console.log('Creating Contexts');
const contexts = [];
const searchTerm = searchDoc.location_name.toLowerCase();
for (const index of matchedIndexes) {
let left = index - 25;
let right = index + searchTerm.length + 25;
if (left < 0) {
left = 0;
}
if (right > text.length) {
right = text.length;
}
const context = text.substring(left, right);
contexts.push({
lid: searchDoc.lid,
context_string: context,
fid: fileName,
});
}
return contexts;
}
function search(text: string, searchTerm: string, suffix_array: number[]) {
console.log(`Beginning search for: ${searchTerm}`);
let start = 0;
let end = suffix_array.length;
const matchedIndexes: Array<number> = [];
while (start < end) {
const mid: number = (end - 1) / 2;
const index: number = suffix_array[mid];
const finalIndex: number = index + searchTerm.length;
if (finalIndex <= text.length) {
const substring: string = text.substring(index, finalIndex);
const match: number = searchTerm.localeCompare(substring);
if (match === 0) {
console.log(`Match Found at Index: ${index}`);
matchedIndexes.push(index);
} else if (match < 0) {
end = mid;
} else if (match > 0) {
start = mid;
}
console.log(matchedIndexes);
}
}
if (matchedIndexes.length === 0) {
console.log(`No matches found for search term: ${searchTerm}`);
}
return matchedIndexes;
}
Hopefully the full function provides a bit more context.
I have watched Doug's videos through a few times but I am still coming up against this, I did notice that removing the await that seems to be failing (as in removing the promise all together) seemed to cause an earlier await to fail. This is indicative of it being an issue with promises later on in the function but I cannot for the life of me find the issue, I will keep trying but hopefully that provides some useful context.

You aren't letting the function know when the async operations have finished.
I would guess that you want to collect all of the async operations in an array and wait for all of them to finish before letting the function exit.
(Those youtube videos by Doug, mentioned in the comments above are quite good and do a more thorough job of explaining why)
ie.
const requests = [];
const things = [1,2,3];
for (let index = 0; index < things.length; index++) {
const element = things[index];
const promise = firebase.firestore().push(element);
requests.push(promise);
}
return Promise.all(requests);

How can I update more than 500 docs in Firestore using Batch?

I'm trying to update a field timestamp with the Firestore admin timestamp in a collection with more than 500 docs.
const batch = db.batch();
const serverTimestamp = admin.firestore.FieldValue.serverTimestamp();
db
.collection('My Collection')
.get()
.then((docs) => {
serverTimestamp,
}, {
merge: true,
})
.then(() => res.send('All docs updated'))
.catch(console.error);
This throws an error
{ Error: 3 INVALID_ARGUMENT: cannot write more than 500 entities in a single call
at Object.exports.createStatusError (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\common.js:87:15)
at Object.onReceiveStatus (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:1188:28)
at InterceptingListener._callNext (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:564:42)
at InterceptingListener.onReceiveStatus (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:614:8)
at callback (C:\Users\Growthfile\Desktop\cf-test\functions\node_modules\grpc\src\client_interceptors.js:841:24)
code: 3,
metadata: Metadata { _internal_repr: {} },
details: 'cannot write more than 500 entities in a single call' }
Is there a way that I can write a recursive method which creates a batch object updating a batch of 500 docs one by one until all the docs are updated.
From the docs I know that delete operation is possible with the recursive approach as mentioned here:
https://firebase.google.com/docs/firestore/manage-data/delete-data#collections
But, for updating, I'm not sure how to end the execution since the docs are not being deleted.

I also ran into the problem to update more than 500 documents inside a Firestore collection. And i would like to share how i solved this problem.
I use cloud functions to update my collection inside Firestore but this should also work on client side code.
The solution counts every operation which is made to the batch and after the limit is reached a new batch is created and pushed to the batchArray.
After all updates are completed the code loops through the batchArray and commits every batch which is inside the array.
It is important to count every operation set(), update(), delete() which is made to the batch because they all count to the 500 operation limit.
const documentSnapshotArray = await firestore.collection('my-collection').get();
const batchArray = [];
batchArray.push(firestore.batch());
let operationCounter = 0;
let batchIndex = 0;
documentSnapshotArray.forEach(documentSnapshot => {
const documentData = documentSnapshot.data();
// update document data here...
batchArray[batchIndex].update(documentSnapshot.ref, documentData);
operationCounter++;
if (operationCounter === 499) {
batchArray.push(firestore.batch());
batchIndex++;
operationCounter = 0;
}
});
batchArray.forEach(async batch => await batch.commit());
return;

I liked this simple solution:
const users = await db.collection('users').get()
const batches = _.chunk(users.docs, 500).map(userDocs => {
const batch = db.batch()
userDocs.forEach(doc => {
batch.set(doc.ref, { field: 'myNewValue' }, { merge: true })
})
return batch.commit()
})
await Promise.all(batches)
Just remember to add import * as _ from "lodash" at the top. Based on this answer.

You can use default BulkWriter. This method used 500/50/5 rule.
Example:
let bulkWriter = firestore.bulkWriter();
bulkWriter.create(documentRef, {foo: 'bar'});
bulkWriter.update(documentRef2, {foo: 'bar'});
bulkWriter.delete(documentRef3);
await close().then(() => {
console.log('Executed all writes');
});

As mentioned above, #Sebastian's answer is good and I upvoted that too. Although faced an issue while updating 25000+ documents in one go.
The tweak to logic is as below.
console.log(`Updating documents...`);
let collectionRef = db.collection('cities');
try {
let batch = db.batch();
const documentSnapshotArray = await collectionRef.get();
const records = documentSnapshotArray.docs;
const index = documentSnapshotArray.size;
console.log(`TOTAL SIZE=====${index}`);
for (let i=0; i < index; i++) {
const docRef = records[i].ref;
// YOUR UPDATES
batch.update(docRef, {isDeleted: false});
if ((i + 1) % 499 === 0) {
await batch.commit();
batch = db.batch();
}
}
// For committing final batch
if (!(index % 499) == 0) {
await batch.commit();
}
console.log('write completed');
} catch (error) {
console.error(`updateWorkers() errored out : ${error.stack}`);
reject(error);
}

Explanations given on previous comments already explain the issue.
I'm sharing the final code that I built and worked for me, since I needed something that worked in a more decoupled manner, instead of the way that most of the solutions presented above do.
import { FireDb } from "#services/firebase"; // = firebase.firestore();
type TDocRef = FirebaseFirestore.DocumentReference;
type TDocData = FirebaseFirestore.DocumentData;
let fireBatches = [FireDb.batch()];
let batchSizes = [0];
let batchIdxToUse = 0;
export default class FirebaseUtil {
static addBatchOperation(
operation: "create",
ref: TDocRef,
data: TDocData
): void;
static addBatchOperation(
operation: "update",
ref: TDocRef,
data: TDocData,
precondition?: FirebaseFirestore.Precondition
): void;
static addBatchOperation(
operation: "set",
ref: TDocRef,
data: TDocData,
setOpts?: FirebaseFirestore.SetOptions
): void;
static addBatchOperation(
operation: "create" | "update" | "set",
ref: TDocRef,
data: TDocData,
opts?: FirebaseFirestore.Precondition | FirebaseFirestore.SetOptions
): void {
// Lines below make sure we stay below the limit of 500 writes per
// batch
if (batchSizes[batchIdxToUse] === 500) {
fireBatches.push(FireDb.batch());
batchSizes.push(0);
batchIdxToUse++;
}
batchSizes[batchIdxToUse]++;
const batchArgs: [TDocRef, TDocData] = [ref, data];
if (opts) batchArgs.push(opts);
switch (operation) {
// Specific case for "set" is required because of some weird TS
// glitch that doesn't allow me to use the arg "operation" to
// call the function
case "set":
fireBatches[batchIdxToUse].set(...batchArgs);
break;
default:
fireBatches[batchIdxToUse][operation](...batchArgs);
break;
}
}
public static async runBatchOperations() {
// The lines below clear the globally available batches so we
// don't run them twice if we call this function more than once
const currentBatches = [...fireBatches];
fireBatches = [FireDb.batch()];
batchSizes = [0];
batchIdxToUse = 0;
await Promise.all(currentBatches.map((batch) => batch.commit()));
}
}

Based on all the above answers, I put together the following pieces of code that one can put into a module in JavaScript back-end and front-end to easily use Firestore batch writes, without worrying about the 500 writes limit.
Back-end (Node.js)
// The Firebase Admin SDK to access Firestore.
const admin = require("firebase-admin");
admin.initializeApp();
// Firestore does not accept more than 500 writes in a transaction or batch write.
const MAX_TRANSACTION_WRITES = 499;
const isFirestoreDeadlineError = (err) => {
console.log({ err });
const errString = err.toString();
return (
errString.includes("Error: 13 INTERNAL: Received RST_STREAM") ||
errString.includes("Error: 4 DEADLINE_EXCEEDED: Deadline exceeded")
);
};
const db = admin.firestore();
// How many transactions/batchWrites out of 500 so far.
// I wrote the following functions to easily use batchWrites wthout worrying about the 500 limit.
let writeCounts = 0;
let batchIndex = 0;
let batchArray = [db.batch()];
// Commit and reset batchWrites and the counter.
const makeCommitBatch = async () => {
console.log("makeCommitBatch");
await Promise.all(batchArray.map((bch) => bch.commit()));
};
// Commit the batchWrite; if you got a Firestore Deadline Error try again every 4 seconds until it gets resolved.
const commitBatch = async () => {
try {
await makeCommitBatch();
} catch (err) {
console.log({ err });
if (isFirestoreDeadlineError(err)) {
const theInterval = setInterval(async () => {
try {
await makeCommitBatch();
clearInterval(theInterval);
} catch (err) {
console.log({ err });
if (!isFirestoreDeadlineError(err)) {
clearInterval(theInterval);
throw err;
}
}
}, 4000);
}
}
};
// If the batchWrite exeeds 499 possible writes, commit and rest the batch object and the counter.
const checkRestartBatchWriteCounts = () => {
writeCounts += 1;
if (writeCounts >= MAX_TRANSACTION_WRITES) {
batchIndex++;
batchArray.push(db.batch());
writeCounts = 0;
}
};
const batchSet = (docRef, docData) => {
batchArray[batchIndex].set(docRef, docData);
checkRestartBatchWriteCounts();
};
const batchUpdate = (docRef, docData) => {
batchArray[batchIndex].update(docRef, docData);
checkRestartBatchWriteCounts();
};
const batchDelete = (docRef) => {
batchArray[batchIndex].delete(docRef);
checkRestartBatchWriteCounts();
};
module.exports = {
admin,
db,
MAX_TRANSACTION_WRITES,
checkRestartBatchWriteCounts,
commitBatch,
isFirestoreDeadlineError,
batchSet,
batchUpdate,
batchDelete,
};
Front-end
// Firestore does not accept more than 500 writes in a transaction or batch write.
const MAX_TRANSACTION_WRITES = 499;
const isFirestoreDeadlineError = (err) => {
return (
err.message.includes("DEADLINE_EXCEEDED") ||
err.message.includes("Received RST_STREAM")
);
};
class Firebase {
constructor(fireConfig, instanceName) {
let app = fbApp;
if (instanceName) {
app = app.initializeApp(fireConfig, instanceName);
} else {
app.initializeApp(fireConfig);
}
this.name = app.name;
this.db = app.firestore();
this.firestore = app.firestore;
// How many transactions/batchWrites out of 500 so far.
// I wrote the following functions to easily use batchWrites wthout worrying about the 500 limit.
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
}
async makeCommitBatch() {
console.log("makeCommitBatch");
if (!this.isCommitting) {
this.isCommitting = true;
await this.batch.commit();
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.isCommitting = true;
await this.batch.commit();
this.writeCounts = 0;
this.batch = this.db.batch();
this.isCommitting = false;
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async commitBatch() {
try {
await this.makeCommitBatch();
} catch (err) {
console.log({ err });
if (isFirestoreDeadlineError(err)) {
const theInterval = setInterval(async () => {
try {
await this.makeCommitBatch();
clearInterval(theInterval);
} catch (err) {
console.log({ err });
if (!isFirestoreDeadlineError(err)) {
clearInterval(theInterval);
throw err;
}
}
}, 4000);
}
}
}
async checkRestartBatchWriteCounts() {
this.writeCounts += 1;
if (this.writeCounts >= MAX_TRANSACTION_WRITES) {
await this.commitBatch();
}
}
async batchSet(docRef, docData) {
if (!this.isCommitting) {
this.batch.set(docRef, docData);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.set(docRef, docData);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async batchUpdate(docRef, docData) {
if (!this.isCommitting) {
this.batch.update(docRef, docData);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.update(docRef, docData);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
async batchDelete(docRef) {
if (!this.isCommitting) {
this.batch.delete(docRef);
await this.checkRestartBatchWriteCounts();
} else {
const batchWaitInterval = setInterval(async () => {
if (!this.isCommitting) {
this.batch.delete(docRef);
await this.checkRestartBatchWriteCounts();
clearInterval(batchWaitInterval);
}
}, 400);
}
}
}

No citations or documentation, this code i invented by myself and for me it worked and looks clean, and simple for read and usage. If some one like it, then can use it too.
Better make autotest becose code use private var _ops wich can be changed after packages upgrade. Forexample in old versions its can be _mutations
async function commitBatch(batch) {
const MAX_OPERATIONS_PER_COMMIT = 500;
while (batch._ops.length > MAX_OPERATIONS_PER_COMMIT) {
const batchPart = admin.firestore().batch();
batchPart._ops = batch._ops.splice(0, MAX_OPERATIONS_PER_COMMIT - 1);
await batchPart.commit();
}
await batch.commit();
}
Usage:
const batch = admin.firestore().batch();
batch.delete(someRef);
batch.update(someRef);
...
await commitBatch(batch);

Simple solution
Just fire twice ?
my array is "resultsFinal"
I fire batch once with a limit of 490 , and second with a limit of the lenght of the array ( results.lenght)
Works fine for me :)
How you check it ?
You go to firebase and delete your collection , firebase say you have delete XXX docs , same as the lenght of your array ? Ok so you are good to go
async function quickstart(results) {
// we get results in parameter for get the data inside quickstart function
const resultsFinal = results;
// console.log(resultsFinal.length);
let batch = firestore.batch();
// limit of firebase is 500 requests per transaction/batch/send
for (i = 0; i < 490; i++) {
const doc = firestore.collection('testMore490').doc();
const object = resultsFinal[i];
batch.set(doc, object);
}
await batch.commit();
// const batchTwo = firestore.batch();
batch = firestore.batch();
for (i = 491; i < 776; i++) {
const objectPartTwo = resultsFinal[i];
const doc = firestore.collection('testMore490').doc();
batch.set(doc, objectPartTwo);
}
await batch.commit();
}