I have been trying for two days to figure this out and now I'm just coding in circles.
All the data gets processed but I can not return the data from the s3.upload function outside the .map function. What am I missing?
if (req.files['altImage']) {
let result = '';
let dataArr = new Array();
const fileArr = req.files['altImage'];
fileArr.map(async file => {
const fileName = file.filename;
const filePath = file.path;
const stream = fs.createReadStream(filePath);
const params = {
ACL: 'public-read',
Bucket: process.env.AWS_S3_BUCKET,
Body: stream,
Key: `${folder}/${fileName}`,
};
const s3Upload = s3.upload(params).promise()
s3Upload
.then(async () => {
await fs.unlinkSync(filePath);
result = await Promise.resolve(s3Upload);
// returns the exact data I need
console.log('result', result);
await dataArr.push(result);
})
.catch((err) => {
console.log(err);
});
})
console.log('dataArr', dataArr);
// dataArr is empty
return {message: 'uploaded', data: dataArr};
}
});
This is probably the quickest way to fix your code:
if (req.files['altImage']) {
let result = '';
const fileArr = req.files['altImage'];
const requests = fileArr.map(file => {
const fileName = file.filename;
const filePath = file.path;
const stream = fs.createReadStream(filePath);
const params = {
ACL: 'public-read',
Bucket: process.env.AWS_S3_BUCKET,
Body: stream,
Key: `${folder}/${fileName}`,
};
return s3.upload(params).promise()
.then(async result => {
await fs.unlinkSync(filePath));
return result;
})
.catch((err) => {
console.log(err);
});
});
Promise.all(requests).then(dataArr => res.send({
message: 'uploaded',
data: dataArr
}));
}
Explanation: you are filling dataArr asynchronously (because the s3 requests are asynchronous), so you need to wait for the requests to end before returning it.
Also, with Express.js, you don't simply return the result, but you pass it to the res object which represents the response.
Related
I am trying to create a middleware that receive a form-data and return the fieldname, contentType and the value. So when I send the firts post the data view in the terminal but if I send the same request again doesn't show me the data in the terminal.
And if a toggle the image, the data come show in the terminal
This is my code:
server:
const express = require("express");
const Upes = require("../upes");
const app = express();
const start = new Upes();
app.post("/", start.setup.bind(start), (req, res) => {
res.send("all right");
});
app.listen(3000, () => {
console.log("The server is active");
});
the index of my middleware:
const getData = require("./utils/getData");
const parseContentType = require("./utils/parseContentType");
class Upes {
setup(req, res, next) {
const contentType = parseContentType(req.headers["content-type"]);
if (!contentType) {
throw new Error("Malformed content-type");
}
const SUBTYPES = ["form-data", "x-www-form-urlencoded"];
if (!SUBTYPES.includes(contentType.subtype)) {
throw new Error(
"The subtypes does not match the following subtypes: " + SUBTYPES
);
}
getData(req, contentType.params.boundary, (data) => {
console.log(data);
});
next();
}
}
module.exports = Upes;
The function that receive the data and processes it:
function getData(req, boundary, callback) {
let chunk = "";
let data = [];
req.on("data", (buffer) => {
chunk += buffer.toString();
});
req.on("end", () => {
// Split the chunk in blocks
const blocks = getBlock(chunk, boundary);
blocks.forEach((block) => {
let [params, value] = block.split("\r\n\r\n");
params = params.split(";");
let fieldname = params[1].split("=")[1].replaceAll('"', "");
let contentType = () => {
const condition = params.length === 3;
if (condition) {
let type = params[2].split(":")[1].replace(" ", "");
return type;
}
return "text-plain";
};
const payload = {
fieldname: fieldname,
contentType: contentType(),
value: "", // value.replace("\r\n", "")
};
data.push(payload);
});
callback(data);
});
}
function getBlock(body, boundary) {
boundary = boundary.replaceAll("-", "");
return body.replaceAll("-", "").split(`${boundary}`).slice(1, -1);
}
module.exports = getData;
Send the same request 20 times
I don't know what happend, please can someone help me?
I am trying to save to json the values returned from indeed api. I use indeed-scraper code from github https://github.com/rynobax/indeed-scraper
My code:
... required files ...
const parsedResults = []
indeed.query(queryOptions).then(response => {
response.forEach((res,i) => {
setTimeout(function(){
let url = res.url
let resultCount = 0
console.log(`\n Scraping of ${url} initiated...\n`)
const getWebsiteContent = async (url) => {
try {
const response = await axios.get(url)
const $ = cheerio.load(response.data)
...get scraped data...
parsedResults.push(metadata)
} catch (error) {
exportResults(parsedResults)
console.error(error)
}
}
getWebsiteContent(url)
}
, i*3000);
});
});
const outputFile = 'data.json'
const fs = require('fs');
const exportResults = (parsedResults) => {
fs.writeFile(outputFile, JSON.stringify(parsedResults, null, 4), (err) => {
if (err) {
console.log(err)
}
console.log(`\n ${parsedResults.length} Results exported successfully to ${outputFile}\n`)
})
}
parsedResults is not accessible in last portion of script, so to save as json file.
Any help appreciated!
The script used when trying to get contents from the csv stored in the s3 bucket
const mysql = require("mysql");
const fs = require("fs");
const { google } = require("googleapis");
const AWS = require("aws-sdk");
const client = new AWS.SecretsManager({ region: "eu-west-1" });
const analyticsreporting = google.analyticsreporting("v4");
const csv = require('ya-csv')
const fastCsv = require('fast-csv')
const s3 = new AWS.S3();
const getParams = {
Bucket: 'data',
Key: 'athena_test/nameplate.csv'
};
exports.handler = async (context, event) => {
const data = await s3.getObject(getParams, function (err, data){
if(err){console.log("ERROR: ",err)}
else {return data}
})
console.log(data.Body)
}
the console log returns undefined rather than the contents of the csv
Hey you can try this one:-
const csv = require('#fast-csv/parse');
const s3Stream = await s3.getObject(params).createReadStream();
const data = await returnDataFromCSV();
console.log(data.Body);
const returnDataFromCSV =()=> {
let promiseData = new Promise((resolve, reject) => {
const parser = csv
.parseStream(csvFile, { headers: true })
.on("data", (data) => {
console.log('Parsed Data:-', data);
})
.on("end", ()=> {
resolve("CSV finished here");
})
.on("error",()=> {
reject("if failed");
});
});
try {
return await promiseData;
} catch (error) {
console.log("Get Error: ", error);
return error;
}
}
CreateStream: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Request.html#createReadStream-property
Having an error might be a simple fix though when i console.log(${req.handle} & console.log(${req.email} & console.log(${req} they all return undefined?
below is my code, is there a reason it's not getting the req data?
exports.uploadImage = (req, res) => {
let db = firebase.firestore();
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs')
const busboy = new BusBoy ({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`feildname is ${fieldname}`);
console.log(`filename is ${filename}`);
console.log(`mimetype is ${mimetype}`);
console.log(`req email is ${req.email}`);
//image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//sfsgsdgsdgsdgsdgsdgsd.png
const imageFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = {filepath , mimetype };
file.pipe(fs.createWriteStream(filepath))
})
busboy.on('finish', () => {
//old syntax
///admin.storage().bucket().upload(imageToBeUploaded.filepath, {
admin.storage().bucket(`${config.storageBucket}`).upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl =
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageFileName}?alt=media`
console.log(`this is req = ${req.email}`)
return db.doc(`/users/${req.user.handle}`).update({ imageUrl});
// return db.doc(`/users/xxcell`).update({ imageUrl});
})
.then(() => {
return res.json({message: 'image uploaded successfully'})
})
.catch(err => {
console.error(err);
return res.status(500).json({error: err.code})
});
});
busboy.end(req.rawBody)
};
The request object in express exposes header parameters in the header function, like so:
app.get("/", (req, res) => {
console.log(`${req.header(email)}`)
res.send(200)
})
So I'm triyng to update some ids from a categories tree using TreeModelJS.
after editing I would like to dump the tree to a file in JSON format.
but when outputing other keys from TreeModel gets outputed as well.
How could I output edited tree as JSON (model only)?
I managed to replace other keys values with null and so far I got this:
const axios = require('axios')
const TreeModel = require('tree-model')
const fs = require('fs')
const url = 'https://my-api-uri-for-categories'
const dumpPath = `${process.cwd()}/data/test/categories.json`
const getCategories = async () => {
try {
const response = await axios.get(url)
return response.data.categories
} catch (error) {
console.log('Error reading categories', error)
}
}
const dumpJsonTofile = data => {
try {
console.log('Dumping to file')
console.log(data)
fs.writeFileSync(
dumpPath,
JSON.stringify(data, (k, v) => {
if (k === 'parent' || k === 'config' || k === 'children') return null
else return v
}),
'utf8'
) // write it back
} catch (error) {
console.log('Error dumping categories', error)
}
}
const scraping = async category => {
try {
const response = await axios.get(category.url)
const document = response.data
const json = document.match(/{"searchTerm"(.*);/g)[0]
const data = JSON.parse(json.replace(';', ''))
return data
} catch (error) {
console.log(`Error while scraping category: ${category.name}`, error)
}
}
async function run() {
const categories = await getCategories()
const categoriesTree = new TreeModel({
childrenPropertyName: 'items',
})
const root = categoriesTree.parse({ id: 0, origin: {}, items: categories })
root.walk(async node => {
const category = node.model
console.log(`scraping category: ${category.name}...`)
if (!category.url) return console.log(`skipping (root?)...`)
const data = await scraping(category)
category.id = data.categoryId
})
dumpJsonTofile(root)
}
run()
but that still outputs a Node object like this:
{
"config":null,
"model":{},
"children":null
}
I need to output all the tree showing only the model key value for each item
Try JSON.stringify(root.model).