We Keep Coding

Cant write to files inside of .then()?

I had a question about some code that i had working earlier, but now decides it doesnt want to work at all. Basically, I have a promise that returns me an array of data. I'll explain whats happening at the bottom of the code.
function mkdir(path){
return new Promise(function(resolve,reject){
fs.mkdir(path, { recursive: true },function(err){
if(err){console.log(err)}
})
return resolve()
})
}
function writeFile(file,content){
return new Promise(function(resolve,reject){
fs.writeFile(file, content, function(err){
if(err){console.log(err)}
return resolve()
})
})
}
function rawData(socket){
var mintDataArray = []
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
getdat(i).then(function(r){
//mintDataArray.push(r)
for(o in r){
var dataurls = []
dataurls.push(r[o].discord_url,r[o].magic_eden_url,r[o].twitter_url)
//socket.emit('twitterFollowers',r[o])
const ProjectData = {
"mintDate": 0 || "",
"name":"",
"stock":0,
"links":[],
"mintTime": 0 || "",
"PricePlusDescription":0 || ""
}
if(r[o].mintDate != null){
ProjectData.mintDate = moment.utc(r[o].mintDate).format("MMMM Do")
}else{
ProjectData.mintDate = "No date specified yet"
}
ProjectData.name = r[o].name
ProjectData.stock = r[o].supply
ProjectData.links.push(dataurls)
ProjectData.PricePlusDescription = r[o].price
mintDataArray.push(ProjectData)
}
}).then(function(socket){
//CollectionSorter(mintDataArray)
for(i in mintDataArray){
var data = mintDataArray[i]
//console.log(data) // <----- This prints out the data that needs to be written to files.
var MintDateFolder = __dirname + "/UpcomingCollections/" +data.mintDate
if(!fs.existsSync(MintDateFolder)){
console.log('huh?')
mkdir(__dirname + '/UpcomingCollections/'+data.mintDate)
}
writeFile(MintDateFolder +"/"+data.name,JSON.stringify(data))
}
})
}
//socket.emit('twitterFollowers',mintDataArray)
})
}
So what the code is supposed to do, is check to see if that directory first exists in general. If it doesnt, then create the new directory. Then after that, its supposed to write files to it (not just that directory, but to other directories as well). It doesnt create the directory if it doesnt exists and it doesnt even write to it if i manually create the directory, however it does write to the other directories. I'm really not sure with this one because I had this working earlier, where it was creating the directory if it didn't exist. so i'm not sure what i messed up on.
I recently made mkdir and writefile functions and i thought they were the issue, because when i had this working I was just using fs.mkdir and fs.writefile. However, i went and tried again without those functions and i was still having the same troubles. I thought about making another promise to check if the directory existed but I already have quite a few nested promises.
read() function:
function read(i){
return new Promise(function(resolve,reject){
var r = https.request(options, function (res) {
var data = []
res.on('data', function (d) {
data.push(d)
}).on('end', function () {
var NFTTokenData = []
console.log(`STATUS: ${res.statusCode}`);
var info = Buffer.concat(data)
zlib.gunzip(info,function(err,buf){
var NFTData = []
var x = buf.toString()
var dat = JSON.parse(x)
var collectionList = dat.pageProps.__APOLLO_STATE__
for(keys in collectionList){
if(collectionList[keys].__typename.includes('Nft')){
collections.push(collectionList[keys])
resolve(collections)
}
}
})
})
})
r.end()
})
}
FINAL SOLUTION
function Project(fromApi) {
return {
mintDate: moment.utc(fromApi.mintDate).format("MMMM Do"),
name: fromApi.name,
imageLink:fromApi.project_image_link,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
console.log(dest)
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return new Promise(function(resolve,reject){
[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17].map(i =>
read(i).then(function(r){
var data = []
for(i in r){
if(r[i].__typename == "Nft"){
data.push(r[i])
}
}
data.map(item => {
var project = Project(item)
write(project,"./UpcomingCollections")
})
})
)
})
}
This is a modification of the answer down below!

You can do anything inside of .then – its only role is to sequence functions.
Let's talk about some other issues in your code -
function mkdir(path){
return new Promise(function(resolve,reject){
fs.mkdir(path, { recursive: true },function(err){
if(err){console.log(err)} // ❌ reject(err); do not log
})
return resolve() // ❌ resolve() outside of mkdir callback
})
}
function writeFile(file,content){
return new Promise(function(resolve,reject){
fs.writeFile(file, content, function(err){
if(err){console.log(err)} // ❌ reject(err); do not log
return resolve() // ⚠️ "return" not needed
})
})
}
Did you find it tedious to implement wrappers for each fs function in Node? You will be happy to know the fs/promises module already provides Promise-based APIs for each -
import { mkdir, writeFile } from "fs/promises" // ✅
// mkdir(path[, options])
// Returns: <Promise> Upon success, fulfills with undefined if recursive is false, or the first directory path created if recursive is true.
// writeFile(file, data[, options])
// Returns: <Promise> Fulfills with undefined upon success.
Next we'll look over the other issues in the main program -
function rawData(socket){
var mintDataArray = [] // ⚠️ state "outside" of Promise context
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
getdat(i).then(function(r){
for(o in r){ // ⚠️ global "o"; don't use for..in, use for..of instead
// ,,,
mintDataArray.push(Project) // ⚠️ attempting to send state "out" of Promise
}
// ⚠️ missing "return"
// implicitly returning "undefined"
}).then(function(socket){ // ⚠️ function parameter (socket) receives resolved Promise
for(i in mintDataArray){ // ⚠️ global "i", for..in again, accessing external state
var data = mintDataArray[i]
var MintDateFolder = __dirname + "/UpcomingCollections/" +data.mintDate // ⚠️ use "path" module
if(!fs.existsSync(MintDateFolder)){ // ⚠️ async methods until this point, why Sync here?
console.log('huh?')
mkdir(__dirname + '/UpcomingCollections/'+data.mintDate) // ⚠️ asynchronous
}
writeFile(MintDateFolder +"/"+data.name,JSON.stringify(data)) // ⚠️ attempts write before mkdir completes
}
})
}
socket.emit('twitterFollowers',mintDataArray) // ⚠️ accessing external state
})
}
read and transform data
It's quite a bit of work, but don't worry. By breaking big problems down into small ones, we can work smarter, not harder. We'll start by renaming getdat to read -
read(i).then(r => {
const mintData = [] // state *inside* promise context
for (const v of r) {
// for all v of r, add project data to mintData
mintData.push({
mintDate: v.mintDate,
name: v.name,
stock: v.supply,
links: [
v.discord_url,
v.magic_eden_url,
v.twitter_url
],
price: v.price
})
}
return mintData // "return" resolves the promise
}).then(...)
Already our .then function is getting big. There's quite a bit of code for extracting and constructing the project data, so make that its own function, Project -
function Project(fromApi) {
// add date logic, or more
return {
mintDate: fromApi.mintDate,
name: fromApi.name,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
read(i).then(r => {
const mintData = []
for (const v of r) {
mintData.push(Project(v)) // ✅
}
return mintData
}).then(...)
Which is the same thing as -
read(i)
.then(r => r.map(Project)) // ✨ no for loop needed!
.then(...)
write
Let's check back in with your code and see our progress -
function rawData(socket){
// var mintDataArray = [] // ✅ remove external state
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
read(i)
.then(r => r.map(Project)) // ✨
.then(function(socket) {
// ⚠️ remember, "socket" gets the resolved promise
// since `read` resolves an array of Projects, we should rename it
})
}
// ⚠️ we'll come back to this
// socket.emit('twitterFollowers',mintDataArray)
})
}
We continue with the .then(function(socket){ ... }) handler. There's a good amount of code here for creating the path, making a directory, and writing JSON to a file. Let's make that its own function called write -
import { stat, mkdir, writeFile } from "fs/promises"
import { join, resolve, dirname } from "path" // 🔍 unrelated to Promise "resolve"
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name)) // ✅ sanitary path handling
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true})) // create dir if not exists
.then(_ => writeFile(dest, JSON.stringify(project))) // write project JSON data
.then(_ => project) // return project
}
Our rawData function is cleaning up nicely, but we still have an outstanding issue of running async operations inside two separate loops -
function rawData(socket){
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
read(i) //❓ how to resolve promise for each read?
.then(r => r.map(Project))
.then(projects => { // ✅ "projects", not "socket"
for (const p of projects) {
write(p, "./UpcomingCollections") //❓ how to resole promise for each write?
}
// ❓ what should we return?
})
}
// socket.emit('twitterFollowers',mintDataArray)
})
}
promises in a loop
Promise has another function we must become familiar with. Promise.all takes an array of promises and resolves only when all promises have resolved -
function myfunc(x) {
return Promise.resolve(x * 100)
}
Promise.all([myfunc(1), myfunc(2), myfunc(3)]).then(console.log)
// Promise{ [ 100, 200, 300 ] }
Promise.all([1,2,3].map(n => myfunc(n))).then(console.log)
// Promise{ [ 100, 200, 300 ] }
We can use Promise.all to clean up the two loops in rawData. And look at that, we can sequence the data directly into the socket -
function rawData(socket){
return Promise.all(
[1,2,3,4,5,6,7,8,9,10,11,12].map(i =>
read(i).then(r => r.map(Project))
)
)
.then(projects => {
Promise.all(projects.map(p => write(p, "./UpcomingCollections"))) // ✨
})
.then(projects => socket.emit("twitterFollowers", projects)) // ✨
}
all together now
We can see a lot of pain points are eliminated by using Promise-based code in an effective, idiomatic way. Until this point we have not addressed the issue of error handling, but now there's nothing left to say. Because we used Promises correctly, any errors will bubble up and the caller can .catch them and respond appropriately -
import { stat, mkdir, writeFile } from "fs/promises"
import { join, resolve, dirname } from "path"
function Project(fromApi) {
return {
mintDate: fromApi.mintDate,
name: fromApi.name,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return Promise.all(
[1,2,3,4,5,6,7,8,9,10,11,12].map(i =>
read(i).then(r => r.map(Project))
)
)
.then(projects => {
Promise.all(projects.map(p => write(p, "./UpcomingCollections")))
})
.then(projects => socket.emit("twitterFollowers", projects))
}
async..await
Modern JavaScript provides async..await syntax allowing us to blur the lines between synchronous and asynchronous code. This allows us to remove many .then calls, flattens our code, reduces cognitive load, and shares asynchronous values in the same scope -
async function rawData(socket) {
const r = await Promise.all([1,2,3,4,5,6,7,8,9,10,11,12].map(read))
const projects = await Promise.all(
r.flatMap(Project).map(p => write(p, "./UpcomingCollections"))
)
return socket.emit("twitterfollowers", projects)
}
Some of the complications of this program stems from the use of nested data. Promise.all is efficient and runs the promises in parallel, but keeping the data nested in the array makes it a bit harder to work with. To show that async..await truly blurs the line between sync and async, we will bring back the two for..of loops and call await in the loop. This results in a serial order processing but readability is terrific. Maybe your use-case isn't hyper demanding and so this style is completely adequate -
async function rawData(socket) {
for (const i of [1,2,3,4,5,6,7,8,9,10,11,12]) {
const result = await read(i)
for (const r of result) {
const project = Project(r)
await write(project, "./UpcomingCollections")
socket.emit("twitterFollowers", project)
}
}
}

Mulan's answer was correct, I just needed to change the for loop in my other function, to a .map. After that, everything worked perfectly. I had to do some modification to the rawData function as seen below.
function Project(fromApi) {
return {
mintDate: moment.utc(fromApi.mintDate).format("MMMM Do"),
name: fromApi.name,
imageLink:fromApi.project_image_link,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
console.log(dest)
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return new Promise(function(resolve,reject){
[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17].map(i =>
read(i).then(function(r){
var data = []
for(i in r){
if(r[i].__typename == "Nft"){
data.push(r[i])
}
}
data.map(item => {
var project = Project(item)
write(project,"./UpcomingCollections")
})
})
)
})
}

How to send res.json() in Express JS from a for loop

I am trying to send a json response when an API call is triggered from front-end, I'm not able to send the res.json() when I am getting the data from a for loop. Where I am writing a query to search in multiple Tables. I am using RethinkDB.
I want res.json() to send data after the query, But I don't understand what mistake I am doing. :(
Thanks in advence
zeasts
Here is the following code and Fiddle Link too.
const express = require("express");
const router = express.Router();
const moment = require('moment');
const r = require('rethinkdb');
const tableNameDB = ['assets', 'alerts', 'destinations']
router.post('/', (req, res, next) => {
let resData = []
let searchValue = req.body.searchValue,
tableName = req.body.tableName;
newCallForSearch(res, searchValue, resData)
})
function newCallForSearch (res, searchValue, resData){
let anArray = ['captain']
for(var i = 0; i<tableNameDB.length; i++){
let tabName = tableNameDB[i]
r.table(tableNameDB[i]).filter(function(doc) {
return doc.coerceTo('string').match(searchValue);
}).run(rethink_conn, (err, cur) => {
// console.log(cur)
if (err) {
return 0
} else {
cur.toArray((err, result) => {
if (err) {
return 0
} else if (result) {
let Results = []
Results = Object.values(result).slice(0,10)
var newResults = Results.map(function() {
resData = Object.assign({'tableName': tabName},{'data' : result});
anArray.push(resData)
})
}
})
}
})
}
res.status(200);
res.json(anArray);
}
module.exports = router;

RethinkDb is a functional database and so using it in a functional way will yield the least resistance. We can accomplish more by writing less code.
You can use Promise.all to run many subqueries and pass the result to res.send -
const search = (table = "", query = "", limit = 10) =>
r.table(table)
.filter(doc => doc.coerceTo("string").match(query))
.toArray()
.slice(0, limit)
.do(data => ({ table, data }))
const tables =
['assets', 'alerts', 'destinations']
const subqueries =
tables.map(t => search(t, "foo").run(rethink_conn)) // <-- search each table
Promise.all(subqueries) // <-- run all subqueries
.then(result => { // <-- handle success
res.status(200)
res.json(result)
})
.catch(e => { // <-- handle failure
res.status(500)
res.send(e.message)
})
Or use .union to produce a single query -
const search = (table = "", query = "", limit = 10) =>
r.table(table)
.filter(doc => doc.coerceTo("string").match(query))
.toArray()
.slice(0, limit)
.do(data => ({ table, data }))
const searchAll = (tables = [], query = "", limit = 10) =>
tables.reduce
( (r, t) => r.union(search(t, query, limit)) // <-- union
, r.expr([]) // <-- if no tables are supplied, return empty result
)
const tables =
['assets', 'alerts', 'destinations']
searchAll(tables, "foo") // <-- single rethink expr
.run(rethink_conn) // <-- run returns a promise
.then(result => { // <-- handle success
res.status(200)
res.json(result)
})
.catch(e => { // <-- handle failure
res.status(500)
res.send(e.message)
})
I should remark on the proposed use of filter in your original post -
.filter(doc => doc.coerceTo("string").match(query))
This is quick but it is also sloppy. It matches query against any of docs values, but also the doc's keys. And if doc is a complex nested document, it matches them too. User beware.

Recursion and Observable RxJs

I am performing pagination inside and Observable stream.
The pagination is implemented with a cursor and a total count using recursion.
I am able to emit the every page using the following code observer.next(searches);, by the way I would like to use just observable and no promises but I cannot express recursion using RxJs operators.
Any suggestions?
const search = id =>
new Observable(observer => { recursePages(id, observer) })
const recursePages = (id, observer, processed, searchAfter) => {
httpService.post(
"http://service.com/search",
{
size: 50,
...searchAfter ? { search_after: searchAfter } : null,
id,
})
.toPromise() // httpService.post returns an Observable<AxiosResponse>
.then(res => {
const body = res.data;
const searches = body.data.hits.map(search => ({ data: search.data, cursor: search.id }));
observer.next(searches);
const totalProcessed = processed + searches.length;
if (totalProcessed < body.data.total) {
return recursePages(id, observer, totalProcessed, searches[searches.length - 1].cursor);
}
observer.complete();
})
}
// General Observer
incomingMessages.pipe(
flatMap(msg => search(JSON.parse(msg.content.toString()))),
concatAll(),
).subscribe(console.log),

these methods will recursively gather all the pages and emit them in an array. the pages can then be streamed with from as shown:
// break this out to clean up functions
const performSearch = (id, searchAfter?) => {
return httpService.post(
"http://service.com/search",
{
size: 50,
...searchAfter ? { search_after: searchAfter } : null,
id,
});
}
// main recursion
const _search = (id, processed, searchAfter?) => {
return performSearch(id, searchAfter).pipe( // get page
switchMap(res => {
const body = res.data;
const searches = body.data.hits.map(search => ({ data: search.data, cursor: search.id }));
const totalProcessed = processed + searches.length;
if (totalProcessed < body.total) {
// if not done, recurse and get next page
return _search(id, totalProcessed, searches[searches.length - 1].cursor).pipe(
// attach recursed pages
map(nextPages => [searches].concat(nextPages)
);
}
// if we're done just return the page
return of([searches]);
})
)
}
// entry point
// switch into from to emit pages one by one
const search = id => _search(id, 0).pipe(switchMap(pages => from(pages))
if what you really need is all of the pages to emit one by one before they're all fetched, for instance so you can show page 1 as soon as it's available rather than wait on page 2+, then that can be done with some tweaking. let me know.
EDIT: this method will emit one by one
const _search = (id, processed, searchAfter?) => {
return performSearch(id, searchAfter).pipe( // get page
switchMap(res => {
const body = res.data;
const searches = body.data.hits.map(search => ({ data: search.data, cursor: search.id }));
const totalProcessed = processed + searches.length;
if (totalProcessed < body.total) {
// if not done, concat current page with recursive call for next page
return concat(
of(searches),
_search(id, totalProcessed, searches[searches.length - 1].cursor)
);
}
// if we're done just return the page
return of(searches);
})
)
}
const search = id => _search(id, 0)
you end up with an observable structure like:
concat(
post$(page1),
concat(
post$(page2),
concat(
post$(page3),
post$(page4)
)
)
)
and since nested concat() operations reduce to a flattened structure, this structure would reduce to:
concat(post$(page1), post$(page2), post$(page3), post$(page4))
which is what you're after and the requests run sequentially.
it also seems like expand might do the trick as per #NickL 's comment, soemthing like:
search = (id) => {
let totalProcessed = 0;
return performSearch(id).pipe(
expand(res => {
const body = res.data;
const searches = body.data.hits.map(search => ({ data: search.data, cursor: search.id }));
totalProcessed += searches.length;
if (totalProcessed < body.data.total) {
// not done, keep expanding
return performSearch(id, searches[searches.length - 1].cursor);
}
return EMPTY; // break with EMPTY
})
)
}
though I've never used expand before and this is based off some very limited testing of it, but I am pretty certain this works.
both of these methods could use the reduce (or scan) operator to gather results if you ever wanted:
search(id).pipe(reduce((all, page) => all.concat(page), []))

This is my used solution combining the expand and reduce operator
searchUsers(cursor?: string) {
return from(
this.slackService.app.client.users.list({
token: this.configService.get('SLACK_BOT_TOKEN'),
limit: 1,
...(cursor && { cursor }),
}),
);
}
Usage
.......
this.searchUsers()
.pipe(
expand((res) => {
if (!!res.response_metadata.next_cursor) {
return this.searchUsers(res.response_metadata.next_cursor);
}
return EMPTY;
}),
reduce((acc, val) => {
return [...acc, ...val.members];
}, []),
)
.subscribe((users) => {
console.log(JSON.stringify(users));
});
....

Recursive Promise-based directory reading

I have a library that scans a directory for files on a remote server. It returns a Promise like this:
client.scanRemoteDirectory(path)
.then(files => {
console.log(files)
})
I'm trying to write a recursive method to scan directories and subdirectories too. But I'm running into some async issues. My function is like this:
const scanDir(path) {
// Scan the remote directory for files and sub-directories
return client.scanRemoteDirectory(path)
.then(files => {
for (const file of files) {
// If a sub-directory is found, scan it too
if (file.type === 'directory') {
return scanDir(file.path) // Recursive call
}
}
})
}
const scanDir('some/path')
.then(() => {
console.log('done')
})
This works however because of the return in front of the scanDir() recursive method call, this results in the method only scanning the first subdirectory in each directory and skipping the rest.
So for example if the structure is something like this:
/some/path
/some/path/dirA
/some/path/dirA/subdirA
/some/path/dirB
/some/path/dirB/subdirB
The above method will only scan:
/some/path
/some/path/dirA
/some/path/subdirA
It will skip dirB and it's children altogether since the method finds dirA first.
If I simply remove the return from the return scanDir(...) call, then it scans everything just fine. But then my final console.log('done') happens too soon because it's async.
So how do I solve this problem? What is the proper recursive Promise approach where I can still preserve async but also scan every subdirectory recursively?

You might want to use Promise.all in this situation to run your 'sub' promises in parallel, for example:
function scanDir(path) {
return client.scanRemoteDirectory(path)
.then(all => {
const files = all.where(file => file.type !== 'directory);
const dirs = all.where(file => file.type === 'directory);
return Promise.all(dirs.map(dir => scanDir(dir.path)) // Execute all 'sub' promises in parallel.
.then(subFiles => {
return files.concat(subFiles);
});
});
}
Alternatively you could use the reduce function to run your 'sub' promises in sequence:
function scanDir(path) {
return client.scanRemoteDirectory(path)
.then(all => {
const files = all.where(file => file.type !== 'directory);
const dirs = all.where(file => file.type === 'directory);
return dirs.reduce((prevPromise, dir) => { // Execute all 'sub' promises in sequence.
return prevPromise.then(output => {
return scanDir(dir.path)
.then(files => {
return output.concat(files);
});
});
}, Promise.resolve(files));
});
}
Async / await is definitely the easiest solution to read:
async function scanDir(path) {
const output = [];
const files = await client.scanRemoteDirectory(path);
for (const file of files) {
if (file.type !== 'directory') {
output.push(file);
continue;
}
const subFiles = await scanDir(file.path);
output = output.concat(subFiles);
}
return output;
}

I would make the then handler async so you can use await in the loop:
const scanDir(path) {
// Scan the remote directory for files and sub-directories
return client.scanRemoteDirectory(path)
.then(async files => {
for (const file of files) {
// If a sub-directory is found, scan it too
if (file.type === 'directory') {
await scanDir(file.path) // Recursive call
}
}
})
}

Multiple queries end with unpredictably sorted result

There are few hundred documents in my database. Schema is very simple:
var firmsSchema = mongoose.Schema({
name: String,
sections: [String],
});
I want to query documents and iterate over:
{{#each sections}}
{{sectionName}}
{{#each firms}}
{{firmName}}
{{/each}}
{{/each}}
Simple:
const SECTIONS = ['name_one', 'name_two', 'name_three'];
const UNSORTED_SECTION_NAME = 'unsorted';
router.get('/', function(req, res, next) {
var showFirms = function showSection (i, acc) {
if (i < 0) return;
let query = SECTIONS[i] ? {sections: SECTIONS[i]} : {sections: {$nin: SECTIONS}};
let key = SECTIONS[i] || UNSORTED_SECTION_NAME;
Firms.find(query).
then((result) => {
acc.push({
section: key,
firms: result,
});
if (i === SECTIONS.length) {
acc = acc.sort((a, b) => (a.section > b.section));
res.render('template', {
sections: acc,
});
}
}).
then(showSection (i - 1, acc));
}
showFirms(SECTIONS.length, []);
};
Works fine. Except it returns acc randomly and unpredictably sorted. I mean 'name_two' section can follow 'name_one' or vise versa.
I thought .sort() at the end of promises chain would be a silver bullet here and solve all asynchronous problems, but it didn't.
Of course i can sort acc with handlebars helper after i pass it to my template, but it is so ridiculously strange i can't sort it right after all queries have been done in my showFirms function.
Can you give me some advise please?

Look at this remake of your code. Instead of getting the data one by one, we gotta get them on the same time (asynchronously) and then treat the return.
If you have any questions I am here, this code is untested so give me a feedback. This is an example of how you can change your code.
const showFirms = function showSection() {
return new Promise((resolve, reject) => {
// Get the keys for the queries
const keys = SECTIONS.map(x => x || UNSORTED_SECTION_NAME);
// For each sections we gonna call a find request
const promises = SECTIONS.map((x, xi) => {
const query = x ? {
sections: x,
} : {
sections: {
$nin: SECTIONS,
},
};
const key = keys[xi];
return Firms.find(query);
});
// Resolve all promises
Promise.all(promises)
.then((rets) => {
// Use the finds results to build an acc array
const accs = rets.map((x, xi) => ({
section: keys[xi],
firms: x,
}));
// Change the sort -> ;) #comments
const sortedAccs = accs.sort((a, b) => (a.section > b.section));
resolve(sortedAccs);
})
.catch(reject);
});
};
How to use it
showFirms()
.then(accs => res.render('template', {
sections: accs,
}))
.catch(err => console.log(`I have an error ${err.toString()}`));

Based on Grégory NEUTS excellent solution. I simplified some things and made the "other sections case" work. I even dropped out sort functionality. Eventual result returns in order of sections as they were declared in the initial SECTIONS array, so now i can just reorder it to control output.
const showFirms = function () {
return new Promise ((resolve, reject) => {
const extendedSections = SECTIONS.slice();
extendedSections.push(UNSORTED_SECTION_NAME);
const promises = extendedSections.map((section) => {
const unsortedCase = section === UNSORTED_SECTION_NAME;
const query = unsortedCase ? {sections: {$nin: SECTIONS}} : {sections: section};
return Firms.find(query);
})
Promise.all(promises)
.then((allResponces) => {
const sectionsData = allResponces.map((response, i) => ({
section: extendedSections[i],
firms: response,
}));
resolve(sectionsData);
})
.catch(reject);
});
};