We Keep Coding

It doesn't show me the data in the terminal when i send the post request twice

I am trying to create a middleware that receive a form-data and return the fieldname, contentType and the value. So when I send the firts post the data view in the terminal but if I send the same request again doesn't show me the data in the terminal.
And if a toggle the image, the data come show in the terminal
This is my code:
server:
const express = require("express");
const Upes = require("../upes");
const app = express();
const start = new Upes();
app.post("/", start.setup.bind(start), (req, res) => {
res.send("all right");
});
app.listen(3000, () => {
console.log("The server is active");
});
the index of my middleware:
const getData = require("./utils/getData");
const parseContentType = require("./utils/parseContentType");
class Upes {
setup(req, res, next) {
const contentType = parseContentType(req.headers["content-type"]);
if (!contentType) {
throw new Error("Malformed content-type");
}
const SUBTYPES = ["form-data", "x-www-form-urlencoded"];
if (!SUBTYPES.includes(contentType.subtype)) {
throw new Error(
"The subtypes does not match the following subtypes: " + SUBTYPES
);
}
getData(req, contentType.params.boundary, (data) => {
console.log(data);
});
next();
}
}
module.exports = Upes;
The function that receive the data and processes it:
function getData(req, boundary, callback) {
let chunk = "";
let data = [];
req.on("data", (buffer) => {
chunk += buffer.toString();
});
req.on("end", () => {
// Split the chunk in blocks
const blocks = getBlock(chunk, boundary);
blocks.forEach((block) => {
let [params, value] = block.split("\r\n\r\n");
params = params.split(";");
let fieldname = params[1].split("=")[1].replaceAll('"', "");
let contentType = () => {
const condition = params.length === 3;
if (condition) {
let type = params[2].split(":")[1].replace(" ", "");
return type;
}
return "text-plain";
};
const payload = {
fieldname: fieldname,
contentType: contentType(),
value: "", // value.replace("\r\n", "")
};
data.push(payload);
});
callback(data);
});
}
function getBlock(body, boundary) {
boundary = boundary.replaceAll("-", "");
return body.replaceAll("-", "").split(`${boundary}`).slice(1, -1);
}
module.exports = getData;
Send the same request 20 times
I don't know what happend, please can someone help me?

Returning Output from AWS.DynamoDB.DocumentClient.Scan() Call

I've got a function that returns the number of records from a DynamoDB table (Things):
const table = 'Things';
const region = 'us-east-1';
const profile = 'development';
process.env.AWS_SDK_LOAD_CONFIG = true;
process.env.AWS_PROFILE = profile;
const AWS = require('aws-sdk');
AWS.config.update({ region: region });
function ddb_table_has_records(table_name) {
const ddb_client = new AWS.DynamoDB.DocumentClient();
const ddb_query_parameters = {
TableName: table_name,
Select: 'COUNT'
}
const results = ddb_client.scan(ddb_query_parameters).promise();
results.then((data) => {
console.log(data.Count);
return data;
}).catch((err) => {
console.log("Error: ", err);
})
}
console.log(ddb_table_has_records(table));
When I run this code, I get the following...
PS C:\> node .\get-count-thing.js
undefined
3951
I'm not capturing the data from the scan in the following; although, I see it in the console.log() call:
console.log(ddb_table_has_records(table));
What am I mucking up?

Posting my fix in-case anyone has the same question. I had to make two changes to retrieve the items from the table; I needed to...
...project ALL_ATTRIBUTES
...iterate over the collection of Items returned
The following was my function with changes:
function ddb_table_has_records(table_name) {
const ddb_client = new AWS.DynamoDB.DocumentClient();
const ddb_query_parameters = {
TableName: table_name,
Select: 'ALL_ATTRIBUTES'
}
const results = ddb_client.scan(ddb_query_parameters).promise();
results.then((data) => {
console.log(data.Count);
data.Items.forEach((thing) => {
console.log(thing);
});
}).catch((err) => {
console.log("Error: ", err);
})
}

Firebase Cloud Functions: Promise not resolving

I am having some issues with my a particular call in my cloud function that doesn't seem to be resolving correctly.
This is the code that doesn't want to resolve:
console.log('Getting Search Patterns');
let searchPatterns: FirebaseFirestore.QuerySnapshot;
try {
searchPatterns = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
} catch (error) {
console.error(error);
}
console.log(`Search Patterns Received: ${searchPatterns}`);
LOG:
As you can see in the log, my function runs up until the console log before the try block, then stops until the function times out. I'm not sure what it is that is causing this issue.
EDIT: I have reformatted my code by separating out each of the different parts in my cloud function into separate functions that I can call; the resulting getSearchTerms() function is as follows:
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
This still stops at the same point in the function execution, the full function is here, this has been updated to the latest version:
import * as functions from 'firebase-functions';
import * as admin from 'firebase-admin';
import * as path from 'path';
import * as WordExtractor from 'word-extractor';
import * as textract from 'textract';
import suffixArray from './suffixArray';
// interface Location {
// lid: string;
// location_name: string;
// location_type: string;
// sentimental_value: number;
// }
// interface Context {
// lid: string;
// context_string: string;
// fid: string;
// }
export const processFile = functions.storage.object().onFinalize(async file => {
const serviceAccount = require(__dirname + '/../config/serviceAccount.json');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: 'https://fyp-alex.firebaseio.com',
});
const firestore = admin.firestore();
const fileBucket: string = file.bucket;
const filePath: string = file.name;
const fileDet: string = path.basename(filePath);
const fileNameSplit: string[] = fileDet.split('.');
const fileExt: string = fileNameSplit.pop();
const fileName: string = fileNameSplit.join('.');
const bucket = admin.storage().bucket(fileBucket);
const fileRef = bucket.file(filePath);
const _path: string = `/tmp/${fileName}.${fileExt}`;
console.log(`File path ${filePath}`);
console.log('Getting Download URL');
try {
console.log(`Downloading to: ${_path}`);
await fileRef.download({ destination: _path });
console.log('File Saved');
console.log(`Getting Details: ${_path}`);
const text: string = await getText(_path, fileExt);
console.log(`Processing: ${fileName}`);
console.log('Creating Suffix Array');
const suffix_array = suffixArray(text);
console.log(`Suffix Array Created: ${suffix_array}`);
console.log('Getting Search Patterns');
const searchTerms: FirebaseFirestore.DocumentData[] = await getSearchTerms();
console.log('Search Patterns Received');
const promises = [];
const allContexts: Object[] = [];
for (const searchDoc of searchTerms) {
const searchTerm = searchDoc.location_name.toLowerCase();
console.log(searchTerm);
const matchedIndexes = search(text, searchTerm, suffix_array);
const contexts = createContexts(matchedIndexes, searchDoc, text, fileName);
allContexts.concat(contexts);
}
for (const context of allContexts) {
const p = admin
.firestore()
.collection('FYP_CONTEXTS')
.add(context);
promises.push(p);
}
await Promise.all(promises);
const data = {
processed: 1,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
} catch (e) {
console.error(e);
const data = {
processed: 2,
};
return firestore.doc(`FYP_FILES/${fileName}`).update(data);
}
});
async function getText(_path: string, fileExt: string) {
let text: string = '';
switch (fileExt) {
case 'docx':
case 'doc':
const extractor = new WordExtractor();
const extracted = await extractor.extract(_path);
text = extracted.getBody();
break;
case 'pdf':
break;
case 'txt':
textract.fromFileWithPath(_path, function(extractedError: any, string: string) {
if (extractedError) {
console.error(extractedError);
}
if (string !== null) {
text = string;
}
});
break;
default:
console.log('Unsupported File Type');
}
return text;
}
async function getSearchTerms(): Promise<FirebaseFirestore.DocumentData[]> {
try {
const snapshot = await admin
.firestore()
.collection('FYP_LOCATIONS')
.get();
console.log('Get Returned');
return snapshot.docs.map(doc => doc.data());
} catch (e) {
console.error(e);
return [];
}
}
function createContexts(
matchedIndexes: number[],
searchDoc: FirebaseFirestore.DocumentData,
text: string,
fileName: string
) {
console.log('Creating Contexts');
const contexts = [];
const searchTerm = searchDoc.location_name.toLowerCase();
for (const index of matchedIndexes) {
let left = index - 25;
let right = index + searchTerm.length + 25;
if (left < 0) {
left = 0;
}
if (right > text.length) {
right = text.length;
}
const context = text.substring(left, right);
contexts.push({
lid: searchDoc.lid,
context_string: context,
fid: fileName,
});
}
return contexts;
}
function search(text: string, searchTerm: string, suffix_array: number[]) {
console.log(`Beginning search for: ${searchTerm}`);
let start = 0;
let end = suffix_array.length;
const matchedIndexes: Array<number> = [];
while (start < end) {
const mid: number = (end - 1) / 2;
const index: number = suffix_array[mid];
const finalIndex: number = index + searchTerm.length;
if (finalIndex <= text.length) {
const substring: string = text.substring(index, finalIndex);
const match: number = searchTerm.localeCompare(substring);
if (match === 0) {
console.log(`Match Found at Index: ${index}`);
matchedIndexes.push(index);
} else if (match < 0) {
end = mid;
} else if (match > 0) {
start = mid;
}
console.log(matchedIndexes);
}
}
if (matchedIndexes.length === 0) {
console.log(`No matches found for search term: ${searchTerm}`);
}
return matchedIndexes;
}
Hopefully the full function provides a bit more context.
I have watched Doug's videos through a few times but I am still coming up against this, I did notice that removing the await that seems to be failing (as in removing the promise all together) seemed to cause an earlier await to fail. This is indicative of it being an issue with promises later on in the function but I cannot for the life of me find the issue, I will keep trying but hopefully that provides some useful context.

You aren't letting the function know when the async operations have finished.
I would guess that you want to collect all of the async operations in an array and wait for all of them to finish before letting the function exit.
(Those youtube videos by Doug, mentioned in the comments above are quite good and do a more thorough job of explaining why)
ie.
const requests = [];
const things = [1,2,3];
for (let index = 0; index < things.length; index++) {
const element = things[index];
const promise = firebase.firestore().push(element);
requests.push(promise);
}
return Promise.all(requests);

Node JS: How to Promise For Loop

I'm trying to create a NodeJS app. Below is code that is supposed to be called when an admin creates a new product. Most of the code works, but I'm having trouble rendering the view only after the rest of the code has executed (the DB functions are asynchronous). I've wrapped much of the code in promises (to make certain blocks execute in the right order) and console logs (to pinpoint problems).
I'd like to point out that the console.dir(rejProducts)just below console.log(111) logs and empty array. Also, adding console.dir(rejProducts) just before the end bracket of the for loop logs an empty array. Thanks! Please let me know if you need more information.
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
Category.find(function(err, categories) {
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
var promiseLoopProducts = new Promise(function(resolve, reject) {
var promiseProducts = [];
// Loop through all addded products
for (let i = 0; i < numProducts; i++) {
var promiseProductCheck = new Promise(function(resolve, reject) {
var name = validate.sanitize(req.body[`name_${i}`]);
var category = validate.sanitize(req.body[`category_${i}`]);
var price = validate.sanitize(req.body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(req.body[`stock_${i}`]);
var image = validate.sanitize(req.body[`image_${i}`]);
var description = validate.sanitize(req.body[`description_${i}`]);
var rejProduct;
var rejFields = { 'name': name, 'category': category, 'price': price,
'stock': stock, 'image': image,
'description': description };
var rejErrors = {};
var productData = {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
var promiseCategoryCheck = new Promise(function(resolve, reject) {
if (ObjectId.isValid(category)) {
var promiseCategoryCount = new Promise(function(resolve, reject) {
Category.count({'_id': category}, function(error, count) {
rejErrors['name'] = validate.isEmpty(name);
if (count == 0) rejErrors['category'] = true;
rejErrors['price'] = !validate.isPrice(price);
rejErrors['stock'] = !validate.isInt(stock);
if( validate.isEmpty(name) || !validate.isPrice(price) ||
count == 0 || !validate.isInt(stock) ) {
rejProduct = { 'fields': rejFields, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
console.log(count);
return resolve();
}
else {
Product.create(productData, function (error, product) {
if (error) return next(error);
console.log(77);
console.dir(rejProducts);
return resolve();
});
}
if (error) return next(error);
});
});
promiseCategoryCount.then(function() {
console.dir(rejProducts);
return resolve();
});
} else {
rejErrors['category'] = true;
rejProduct = { 'fields': rejFields, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
}
});
promiseCategoryCheck.then(function() {
console.dir(rejProducts);
promiseProducts.push(promiseProductCheck);
console.log(promiseProductCheck);
console.log(promiseProducts);
return resolve();
});
});
promiseProductCheck.then(function() {
console.log(106);
console.dir(rejProducts);
});
}
Promise.all(promiseProducts).then(function() {
console.log(111);
console.dir(rejProducts); // Empty array!
return resolve();
});
});
promiseLoopProducts.then(function() {
console.log(118);
console.dir(rejProducts); // Empty array!
res.render('products/new', { categories: categories, rejProducts: rejProducts });
});
});
});
Edit: I've made some changes to the code. There it seems like util.promisify is not being recognized as a function. I am using Node 9.4.
module.exports = function(app){
const util = require('util');
require('util.promisify').shim();
const validate = require('../functions/validate');
const Category = require('../db/categories');
const Product = require('../db/products');
var ObjectId = require('mongoose').Types.ObjectId;
//Category.find as function that returns a promise
const findCategories = util.promisify(Category.find);
const countCategories = (categoryId) => {
util.promisify(Category.count({'_id': categoryId}));
};
const bodyToProduct = (body, i) => {
var name = validate.sanitize(body[`name_${i}`]);
var category = validate.sanitize(body[`category_${i}`]);
var price = validate.sanitize(body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(body[`stock_${i}`]);
var image = validate.sanitize(body[`image_${i}`]);
var description = validate.sanitize(body[`description_${i}`]);
return {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
};
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
return findCategories()
.then(
categories=>{
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
return Promise.all(
Array.from(new Array(numProducts),(v,i)=>i)
.map(//map [0...numProducts] to product object
i=>bodyToProduct(req.body,i)
)
.map(
product => {
var rejErrors;
var rejName = validate.isEmpty(name);
var rejCategory;
var rejPrice = !validate.isPrice(price);
var rejStock = !validate.isInt(stock);
if (ObjectId.isValid(product.category)) {
return countCategories()
.then(
count=> {
if (count == 0) rejCategory = true;
if(rejName || rejCategory || rejPrice || rejStock ) {
rejErrors = {
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
rejProduct = { 'fields': product, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
console.log(count);
} else {
Product.create(productData, function (error, product) {
if (error) return next(error);
console.log(77);
console.dir(rejProducts);
});
}
}
).catch(function() {
console.log("Count function failed.");
});
} else {
rejCategory = true;
rejErrors = {
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
rejProduct = { 'fields': product, 'errors': rejErrors };
rejProducts.push(rejProduct);
console.dir(rejProducts);
}
}
)
).then(function() {
res.render('products/new', { categories: categories, rejProducts: rejProducts });
}).catch(function() {
console.log("Promise all products failed.");
});
}
).catch(function() {
console.log("Find categories failed.");
})
});
}

Some tips: if your function is over 100 lines you may be doing to much in the function.
If you have to get data from your request the way you get products of it then write better client side code (products should be an array of product objects that should not need to be sanitized). Validation is needed on the server because you an never trust what the client is sending you. But looking at the sanitize you don't even trust what your client script is sending you.
Try to write small functions that do a small thing and try to use those.
Use .map to map type a to type b (for example req.body to array of products as in the example code).
Use the result of .map as an argument to Promise.all
Use util.promisify to change a callback function into a function that returns a promise, since you are using an old version of node I've added an implementation of promisify:
var promisify = function(fn) {
return function(){
var args = [].slice.apply(arguments);
return new Promise(
function(resolve,reject){
fn.apply(
null,
args.concat([
function(){
var results = [].slice.apply(arguments);
(results[0])//first argument of callback is error
? reject(results[0])//reject with error
: resolve(results.slice(1,results.length)[0])//resolve with single result
}
])
)
}
);
}
};
module.exports = function(app){
const validate = require('../functions/validate');
const Category = require('../db/categories');
const Product = require('../db/products');
var ObjectId = require('mongoose').Types.ObjectId;
//Category.find as function that returns a promise
const findCategories = promisify(Category.find.bind(Category));
const countCategories = (categoryId) => {
promisify(Category.count.bind(Category))({'_id': categoryId});
};
const createProduct = promisify(Product.create.bind(Product));
const REJECTED = {};
const bodyToProduct = (body, i) => {
var name = validate.sanitize(body[`name_${i}`]);
var category = validate.sanitize(body[`category_${i}`]);
var price = validate.sanitize(body[`price_${i}`].replace(/\$/g, ""));
var stock = validate.sanitize(body[`stock_${i}`]);
var image = validate.sanitize(body[`image_${i}`]);
var description = validate.sanitize(body[`description_${i}`]);
return {
name: name,
category: category,
price: price,
stock: stock,
image: image,
description: description
};
};
const setReject = product => {
var rejErrors;
var rejName = validate.isEmpty(product.name);
var rejCategory;
var rejPrice = !validate.isPrice(product.price);
var rejStock = !validate.isInt(product.stock);
const countPromise = (ObjectId.isValid(product.category))
? countCategories()
: Promise.resolve(0);
return countPromise
.then(
count => {
if (count == 0) rejCategory = true;
if (rejName || rejCategory || rejPrice || rejStock) {
rejErrors = {
type:REJECTED,
name: rejName,
category: rejCategory,
price: rejPrice,
stock: rejStock
}
return rejErrors;
}
return false;
}
);
};
const productWithReject = product =>
Promise.all([
product,
setReject(product)
]);
const saveProductIfNoRejected = ([product,reject]) =>
(reject===false)
? Product.create(product)
.catch(
err=>({
type:REJECTED,
error:err
})
)
: reject;
app.post('/products/new', function (req, res, next) {
// Async function: find all categories
return findCategories()
.then(
categories => {
// Hidden count that tells num products to be created by form
var numProducts = req.body[`form-item-count`];
// Array of all rejected products adds
var rejProducts = [];
return Promise.all(
Array.from(new Array(numProducts), (v, i) => i)
.map(//map [0...numProducts] to product object
i => bodyToProduct(req.body, i)
)
.map(
product=>
productWithReject(product)
.then(saveProductIfNoRejected)
)
).then(
results =>
res.render(
'products/new',
{
categories: categories,
rejProducts: results.filter(x=>(x&&x.type)===REJECTED)
}
)
).catch(
error=>
console.log("Promise all products failed.",error)
);
}
).catch(
error=>
console.log("Find categories failed.",error)
)
});
}

Unexpected token o in JSON: How to fix?

I'm trying to save member data as JSON, but it is not working. I'm using discord.js framework.
I have searched Stack Overflow and many other sites, but I can't find a solution.
Here's the code snippet:
const user = message.guild.members.get(message.mentions.users.first().id);
const file = (user.id + ".json");
const content = {
"name": user.username,
"isFlagged": false,
"punishments": 1,
"id": user.id,
"discrim": user.discriminator
};
var B = content.toString();
fs.writeFileIfNotExist(file, B, function(err, existed) {
if (err) {
console.log(err)
} else {
console.log('file created sucessfuly!')
}
});
let points = JSON.parse(fs.readFileSync('./' + user.id + '.json'));
points.isFlagged = false;
points.punishments = points.punishments + 1;

As, CRice notes, calling Object.prototype.toString on an object created from a literal and not shadowing toString with an implementation returning a JSON formatted string, will return the string "[object Object"] which is not what you want.
This is demonstrated by the following snippet:
const content = {
name: 'John Smith',
isFlagged: false,
punishments: 1,
id: 5,
discrim: Symbol()
};
const b = content.toString();
console.log(b);
So first, change your code as follows
const content = {
name: 'John Smith',
isFlagged: false,
punishments: 1,
id: 5,
discrim: Symbol()
};
const b = JSON.stringify(content);
console.log(b);
Lastly, consider rewriting your program to cleanly and consistently leverage asynchronous IO and improve your naming patterns to follow established JavaScript conventions.
const promisify = require('util.promisify');
const fs = require('fs');
const exists = promisify(fs.exists);
const writeFile = promisify(fs.writeFile);
const readFile = promisify(fs.readFile);
const user = message.guild.members.get(message.mentions.users.first().id);
const file = `${user.id}.json`;
const content = {
name: user.username,
isFlagged: false,
punishments: 1,
id: user.id,
discrim: user.discriminator
};
const b = JSON.stringify(content);
(async function () {
try {
await writeFileIfNotExist(file, b);
console.log('file created sucessfuly!')
} catch (e) {
console.error(e);
}
const json = await readFile(`./${user.id}.json`);
const user = JSON.parse(json);
user.isFlagged = false;
user.punishments = user.punishments + 1;
}());
async function writeFileIfNotExists(fileName, data) {
if (await exists(fileName)) {
return;
}
try {
await writeFile(fileName, data);
} catch (e) {
console.error(e);
}
}