how to get all Images exif and pass it to view in Expressjs
basically it has to wait for 2 promisses
get filnames with readdir and get exifs and return as array to view
here what I have but lost.
const express = require('express');
const exifr = require('exifr');
const fs = require('fs');
const fsPromises = fs.promises;
const app = express();
app.set('view engine', 'ejs')
let defaultOptions = {
// Segments (JPEG APP Segment, PNG Chunks, HEIC Boxes, etc...)
tiff: false,
xmp: false,
icc: false,
iptc: true,
jfif: false, // (jpeg only)
ihdr: false, // (png only)
// Sub-blocks inside TIFF segment
ifd0: false, // aka image
ifd1: false, // aka thumbnail
exif: false,
gps: true,
}
const photosDir = './photos/'
app.use('/photos', express.static('photos'))
async function listDir() {
try {
return fsPromises.readdir(photosDir) .then(filenames => {
for (let filename of filenames) {
//console.log(filename)
exifr.parse(photosDir+filename, defaultOptions).then(function(data){
console.log(data)
return data
})
}
});
} catch (err) {
console.error('Error occured while reading directory!', err);
}
}
listDir()
app.get('/', async (_,res) => {
let data = await listDir()
console.log(data)
res.render('index',{"files": data })
});
app.listen(3000, () => console.log('Example app is listening on port 3000.'));
Use Promise.all, this will resolve after all exifr.parse promises have resolved.
return fsPromises.readdir(photosDir)
.then(filenames => Promise.all(filenames.map(filename =>
exifr.parse(photosDir+filename, defaultOptions)
.then(function(data) {
console.log(data)
return data
})
)));
Related
Based on this question, I need to also make a test for a middleware which also uses the db-connection.js file. The middleware file will look like this:
const dbConnection = require('./db-connection.js')
module.exports = function (...args) {
return async function (req, res, next) {
// somethin' somethin' ...
const dbClient = dbConnection.db
const docs = await dbClient.collection('test').find()
if (!docs) {
return next(Boom.forbidden())
}
}
}
, the database connection file do not change, which is:
const MongoClient = require('mongodb').MongoClient
const dbName = 'test'
const url = process.env.MONGO_URL
const client = new MongoClient(url, { useNewUrlParser: true,
useUnifiedTopology: true,
bufferMaxEntries: 0 // dont buffer querys when not connected
})
const init = () => {
return client.connect().then(() => {
logger.info(`mongdb db:${dbName} connected`)
const db = client.db(dbName)
})
}
/**
* #type {Connection}
*/
module.exports = {
init,
client,
get db () {
return client.db(dbName)
}
}
How the middleware works is by passing list of strings (that strings is roles), I have to query to the database and check whether there is a record of each roles. If the record exists, I will return next(), while if the record does not exist, I will return next(Boom.forbidden()) (next function with a 403 status code from Boom module).
Given the details above, how does one make a test to test out the return value of the middleware if the record exists or not? This means I have to assert the next() and next(Boom.forbidden) to be exact.
Based on the answer. You can create stubs for the req, res objects, and next function.
E.g.(Doesn't run, but it should work.)
const sinon = require('sinon');
describe('a', () => {
afterEach(() => {
sinon.restore();
});
it('should find some docs', async () => {
process.env.MONGO_URL = 'mongodb://localhost:27017';
const a = require('./a');
const dbConnection = require('./db-connection.js');
const dbStub = {
collection: sinon.stub().returnsThis(),
find: sinon.stub(),
};
sinon.stub(dbConnection, 'db').get(() => dbStub);
const req = {};
const res = {};
const next = sinon.stub();
const actual = await a()(req, res, next);
sinon.assert.match(actual, true);
sinon.assert.calledWithExactly(dbStub.collection, 'test');
sinon.assert.calledOnce(dbStub.find);
sinon.assert.calledOnce(next);
});
});
How to bufferize efficiently in nodeJS on events from a stream to bulk insert instead of unique insert per record received from the stream. Here's pseudo code I've got in mind:
// Open MongoDB connection
mystream.on('data', (record) => {
// bufferize data into an array
// if the buffer is full (1000 records)
// bulk insert into MongoDB and empty buffer
})
mystream.on('end', () => {
// close connection
})
Does this look realistic?
Is there any possible optimization? Existing libraries facilitaties that?
Using NodeJS' stream library, this can be concisely and efficiently implemented as:
const stream = require('stream');
const util = require('util');
const mongo = require('mongo');
const streamSource; // A stream of objects from somewhere
// Establish DB connection
const client = new mongo.MongoClient("uri");
await client.connect();
// The specific collection to store our documents
const collection = client.db("my_db").collection("my_collection");
await util.promisify(stream.pipeline)(
streamSource,
stream.Writable({
objectMode: true,
highWaterMark: 1000,
writev: async (chunks, next) => {
try {
const documents = chunks.map(({chunk}) => chunk);
await collection.insertMany(docs, {ordered: false});
next();
}
catch( error ){
next( error );
}
}
})
);
I ended up with a no dependency solution.
const { MongoClient } = require("mongodb")
const url = process.env.MONGO_URI || "mongodb://localhost:27019";
const connection = MongoClient.connect(url, { useNewUrlParser: true, useUnifiedTopology: true })
Promise.resolve(connection)
.then((db) => {
const dbName = "databaseName";
const collection = 'collection';
const dbo = db.db(dbName);
let buffer = []
stream.on("data", (row: any) => {
buffer.push(row)
if (buffer.length > 10000) {
dbo.collection(collection).insertMany(buffer, {ordered: false});
buffer = []
}
});
stream.on("end", () => {
// insert last chunk
dbo.collection(collection).insertMany(buffer, {ordered: false})
.then(() => {
console.log("Done!");
db.close();
})
});
stream.on("error", (err) => console.log(err));
})
.catch((err) => {
console.log(err)
})
Having an error might be a simple fix though when i console.log(${req.handle} & console.log(${req.email} & console.log(${req} they all return undefined?
below is my code, is there a reason it's not getting the req data?
exports.uploadImage = (req, res) => {
let db = firebase.firestore();
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs')
const busboy = new BusBoy ({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`feildname is ${fieldname}`);
console.log(`filename is ${filename}`);
console.log(`mimetype is ${mimetype}`);
console.log(`req email is ${req.email}`);
//image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//sfsgsdgsdgsdgsdgsdgsd.png
const imageFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = {filepath , mimetype };
file.pipe(fs.createWriteStream(filepath))
})
busboy.on('finish', () => {
//old syntax
///admin.storage().bucket().upload(imageToBeUploaded.filepath, {
admin.storage().bucket(`${config.storageBucket}`).upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl =
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageFileName}?alt=media`
console.log(`this is req = ${req.email}`)
return db.doc(`/users/${req.user.handle}`).update({ imageUrl});
// return db.doc(`/users/xxcell`).update({ imageUrl});
})
.then(() => {
return res.json({message: 'image uploaded successfully'})
})
.catch(err => {
console.error(err);
return res.status(500).json({error: err.code})
});
});
busboy.end(req.rawBody)
};
The request object in express exposes header parameters in the header function, like so:
app.get("/", (req, res) => {
console.log(`${req.header(email)}`)
res.send(200)
})
Trying to find out why my scrape function runs twice. I am trying to create a SSR app that hits a website and scrapes the data. It does work and correctly grabs the data I need but I set headless to false and I can see the browser opening correctly and then opening again and searching for the term [object Object]...
I need it to only run on the server that is why I have gone the express + next.js combo.
server.js
const express = require("express");
const next = require("next");
const scraper = require("./utils/scraper");
const port = parseInt(process.env.PORT, 10) || 3000;
const dev = process.env.NODE_ENV !== "production";
const nextApp = next({ dev });
const nextHandle = nextApp.getRequestHandler();
nextApp.prepare().then(() => {
const server = express();
server.get("/search/:query", async (req, res) => {
const { query } = req.params;
const listings = await scraper.scrape(query);
return nextApp.render(req, res, "/search", { search: query, listings });
});
server.get("*", (req, res) => {
return nextHandle(req, res);
});
server.listen(port, err => {
if (err) {
throw err;
}
console.log(`> Ready on http://localhost:${port}`);
});
});
scraper.js
const puppeteer = require("puppeteer");
const scrape = async term => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto(`https://...&query=${term}`);
const scrapedData = await page.evaluate(() => {
const items = document.querySelectorAll(".results");
return Array.from(items).map(listing => {
return { ... build up my obj here };
});
});
await browser.close();
return scrapedData;
};
module.exports.scrape = scrape;
Search.jsx (the next.js page)
import React, { Component } from "react";
export default class extends Component {
static async getInitialProps(ctx) {
return {
search: ctx.query.search,
listings: ctx.query.listings
};
}
render() {
const { search, listings } = this.props;
console.log(listings, "client");
return (
<div>
<h1>{search} search!</h1>
{ ...listings.map() }
</div>
);
}
}
UPDATE 1
I noticed that puppeteer will correctly only open once if I do not pass my listings array into the nextApp.render and just log out the results on the server. But as soon as I pass it along to the page to to getInitialProps I experience the double loading as explained above.
Scenario[UPDATED]
I'm trying to connect to mongodb before running test cases and if I'm not wrong I can use beforeAll which is included in Jest where I can connect to my DB before running test cases, I am also testing my REST api with it
Test
const request = require ('supertest');
const app = require ('../../app');
const db = require ('../../db.js');
const url = 'mongodb://localhost:27017';
//UPDATED beforeALL (thanks to #andreas-köberle)
beforeAll ((done) => {
db.connect (url, (err) => {
if (err) {
console.log ('Unable to connect',err)
process.exit(1)
} else {
console.log('success')
}
});
});
test('should response the GET method', async () => {
console.log('DADAD');
const res = await request (app).get ('/expense'); // I've set /expense in app (app.use('/expense,'expenseRoute)
return expect(res.statusCode).toBe (200);
});
afterAll ( () => {
db.close ();
});
DB
const MongoClient = require ('mongodb').MongoClient;
const dbName = 'expenseTest';
let state = {
db: null,
};
exports.connect = (url, done) => {
if (state.db) return done ();
MongoClient.connect (url, (err, client) => {
const db = client.db(dbName);
state.db = db;
done ();
});
};
exports.get = () => {
return state.db;
};
exports.close = done => {
if (state.db) {
state.db.close ((err, res) => {
state.db = null;
done (err);
});
}
};
ROUTE
const express = require ('express')
const router = express.Router ()
const MongoClient = require ('mongodb').MongoClient
const assert = require ('assert')
let db = require ('../db')
/**
* Returns the expense
*/
router.get ('/', (req, res) => {
console.log(db.get());
let expenseCollection = db.get ().collection ('expenseTrack')
expenseCollection.find({}).toArray((err, docs) => {
res.status(200).send(docs)
})
//res.status(200).send('hello')
})
/**
* Stores the expense in db
*/
router.post ('/', (req, res) => {
let expenseCollection = db.get ().collection ('expenseTrack')
expenseCollection.insert (req.body, (err, result) => {
if (err) console.log (err)
else res.status (200).send (result.ops)
})
})
module.exports = router
I have console logs in Test,in my GET route and in beforeAll, here's the output of npm run test
● Console
console.log test/express/startupTest.test.js:18
DADAD
console.log routes/Expense.js:13
null
console.log test/express/startupTest.test.js:11
Succesfully
So It's clear that it's coming in Test first, If I change my endpoint code to this all test case works fine.
/**
* Returns the expense
*/
router.get ('/', (req, res) => {
// console.log(db.get());
// let expenseCollection = db.get ().collection ('expenseTrack')
// expenseCollection.find({}).toArray((err, docs) => {
// res.status(200).send(docs)
// })
res.status(200).send('hello')
})
After updating beforeAll it is now giving me another error/excpetion
Timeout - Async callback was not invoked within timeout specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.