Problem with parsing data at filestream and cvparser - javascript

I'm having a problem reading this piece of code in a project that we are developing with colleagues. I'll be greateful if you can help me and tell me what that piece of code mean. I get that in that piece, I`m having the technologies.csv file and we need to parse it, and if there is an error we must throw an exception to this error. And at the end of the code there are some cards but i don't know what is the idea of this cards. And at the final we have to export the module with the data, and i think the data is from the const with the programing languages.If someone can explain it with details it would be more than perfect. Thanks in advance! :)
const fs = require('fs');
const parse = require('csv-parse');
const path = require('path');
const constants = {
testData: {
csvColumns: [
'ruby',
'python',
'vuejs',
'angular',
'react',
'nodejs',
],
},
};
const configFileLocation = (name) => {
return
{
filename: path.join(__dirname, `${name}technologies.csv`)
}
}
const getData = (name) =>
new Promise((resolve, reject) => {
const fileLocation = configFileLocation(name).filename;
const csvParser = parse({
delimiter: ',',
});
if (!fs.existsSync(fileLocation)) {
reject(new Error(`File ${fileLocation} is missing.`));
}
const csvFileStream = fs.createReadStream(fileLocation);
csvFileStream.on('ready', () => {
csvFileStream.pipe(csvParser);
});
csvFileStream.on('error', (error) => {
reject(
new Error({
error,
message: 'csvParseCards#csvFileStream on error',
})
);
});
csvParser.on('error', (error) => {
reject(
new Error({
error,
message: 'csvParseCards#csvParser on error',
})
);
});
const cards = [];
csvParser.on('readable', () => {
let record = '';
while ((record = csvParser.read())) {
const card = {};
const columns = constants.testData.csvColumns;
if (record.length !== columns.length) {
console.warn('Column mismatch', record);
}
record.map((value, index) => {
card[columns[index]] = value;
});
cards.push(card);
}
});
csvParser.on('end', () => {
cards.shift();
resolve(cards);
});
});
module.exports = getData;

Related

How to stub a dependency in express middleware (express-validator/mocha/chai/sinon/proxyquire)

**Edit: Re-written with a simple example that works first:
So I've got a test file and 2 modules.
moduleA has a dependency, moduleB
// moduleA.js
const ModuleB = require('./moduleB');
function functionA() {
return 20 + ModuleB.functionB();
};
module.exports = { functionA };
// moduleB.js
const functionB = () => {
return 10;
}
module.exports = { functionB }
My test file stubs out functionB (returned from moduleB) using proxyquire:
const sinon = require('sinon');
const proxyquire = require('proxyquire');
describe('Unit Tests', function() {
it('should work', () => {
const mockedFn = sinon.stub();
mockedFn.returns(30);
const copyModuleA = proxyquire('./moduleA', {
'./moduleB': {
functionB: mockedFn
}
});
console.log(copyModuleA.functionA());
})
});
So it outputs 50 (stubbed functionB 30 + functionA 20)
Now I'm trying to take this example into my code:
moduleA in this case is a file called validation.js. It is dependent on moduleB, in this case a sequelize model, Person, with the function I want to mock: findOne
validation.js exports module.exports = { validateLogin };, a function that calls validate, which returns a function that uses Person.findOne()
So in my mind, as with the simple example, I need to create a stub, point to the validation module in proxyquire, and reference the dependency and its findOne function. Like this:
const stubbedFindOne = sinon.stub();
stubbedFindOne.resolves();
validationModule = proxyquire('../../utils/validation', {
'../models/Person': {
findOne: stubbedFindOne
}
});
This should stub Person.findOne in validation.js. But it doesn't seem to. And I have no idea why.
let validationModule;
describe('Unit Tests', () => {
before(() => {
const stubbedFindOne = sinon.stub();
stubbedFindOne.resolves();
validationModule = proxyquire('../../utils/validation', {
'../models/Person': {
findOne: stubbedFindOne
}
});
})
it.only('should return 422 if custom email validation fails', async() => {
const wrongEmailReq = { body: {email: 'nik#hotmail.com'} };
const res = {
statusCode: 500,
status: (code) => {this.statusCode = code; return this},
};
const validationFn = validationModule.validateLogin();
const wrongEmail = await validationFn(wrongEmailReq, res, ()=>{});
expect(wrongEmail.errors[0].msg).to.be.equal('Custom Authorisation Error');
return;
})
And this is my validation.js file:
const Person = require('../models/Person');
// parallel processing
const validate = validations => {
return async (req, res, next) => {
await Promise.all(validations.map(validation => validation.run(req)));
const errors = validationResult(req);
if (errors.isEmpty()) {
return next();
}
const error = new Error();
error.message = process.env.NODE_ENV === 'development'? 'Validation Failed':'Error';
error.statusCode = !errors.isEmpty()? 422:500;
error.errors = errors.array({onlyFirstError: true});
next(error);
return error;
};
};
const validateLogin = () => {
const validations = [
body('email')
.isString()
// snip
.custom(async (value, {req}) => {
try{
const person = await Person.findOne({ where: { email: value } });
if(!person) return Promise.reject('Custom Authorisation Error');
} catch(err) {
throw err;
}
})
.trim(),
];
return validate(validations);
}
module.exports = {
validateLogin
};
So the code in both the small sample and my app is correct, apart from how I stub the function. It shouldn't resolve or reject anything (I tried both out of desperation). It should return null in order to satisfy the conditional rather than jump to the catch block:
try{
const person = await Person.findOne({ where: { email: value } });
if(!person) return Promise.reject('Custom Authorisation Error');
} catch(err) {
throw err;
}
Hope the simple example helps someone else with proxyquire though

How to initialize App Data in node js and access it without being undefined in jest test?

i am initializing a node js app with crucial data for the app to work from a database in index.js.
index.ts
import {getInitialData} from 'initData.ts';
export let APP_DATA: AppData;
export const initializeAppData = async () => {
try {
APP_DATA = (await getInitialData()) as AppData;
if (process.env.NODE_ENV !== 'test') {
initializeMongoose();
startServer();
}
} catch (error) {
console.log(error);
}
};
initData.ts
let dbName: string = 'initialData';
if (process.env.NODE_ENV === 'test') {
dbName = 'testDb';
}
const uri = `${process.env.MONGODB_URI}/?maxPoolSize=20&w=majority`;
export async function getInitialData() {
const client = new MongoClient(uri);
try {
await client.connect();
const database = client.db(dbName);
const configCursor = database
.collection('config')
.find({}, { projection: { _id: 0 } });
const config = await configCursor.toArray();
const aaoCursor = database
.collection('aao')
.find({}, { projection: { _id: 0 } });
const aao = await aaoCursor.toArray();
return { config, aao };
} catch {
(err: Error) => console.log(err);
} finally {
await client.close();
}
}
I'm using this array in another file and import it there.
missionCreateHandler
import { APP_DATA } from '../index';
export const addMissionResources = (
alarmKeyword: AlarmKeyword,
newMission: MissionDocument
) => {
const alarmKeywordObject = APP_DATA?.aao.find(
(el) => Object.keys(el)[0] === alarmKeyword
);
const resourceCommand = Object.values(alarmKeywordObject!);
resourceCommand.forEach((el) => {
Object.entries(el).forEach(([key, value]) => {
for (let ii = 1; ii <= value; ii++) {
newMission.resources?.push({
initialType: key,
status: 'unarranged',
});
}
});
});
};
I'm setting up a mongodb-memory-server in globalSetup.ts for Jest and copy the relevant data to the database from json-files.
globalSetup.ts
export = async function globalSetup() {
const instance = await MongoMemoryServer.create({
instance: { dbName: 'testDb' },
});
const uri = instance.getUri();
(global as any).__MONGOINSTANCE = instance;
process.env.MONGODB_URI = uri.slice(0, uri.lastIndexOf('/'));
process.env.JWT_SECRET = 'testSECRET';
const client = new MongoClient(
`${process.env.MONGODB_URI}/?maxPoolSize=20&w=majority`
);
try {
await client.connect();
const database = client.db('testDb');
database.createCollection('aao');
//#ts-ignore
await database.collection('aao').insertMany(aao['default']);
} catch (error) {
console.log(error);
} finally {
await client.close();
}
};
missionCreateHandler.test.ts
test('it adds the correct mission resources to the array', async () => {
const newMission = await Mission.create({
address: {
street: 'test',
houseNr: 23,
},
alarmKeyword: 'R1',
});
const expected = {
initialType: 'rtw',
status: 'unarranged',
};
addMissionResources('R1', newMission);
expect(newMission.resources[0].initialType).toEqual(expected.initialType);
expect(newMission.resources[0].status).toEqual(expected.status);
});
When runing the test, i get an 'TypeError: Cannot convert undefined or null to object at Function.values ()'. So it seems that the APP_DATA object is not set. I checked that the mongodb-memory-server is set up correctly and feed with the needed data.
When i hardcode the content of APP_DATA in index.ts, the test runs without problems.
So my questions are: How is the best practice to set up initial data in a node js app and where to store it (global object, simple variable and import it in the files where needed)? How can the test successfully run, or is my code just untestable?
Thank you!

Solana method cannot convert string to Base58 format

I'm using #solana/web3.js and have this code:
const web3 = require("#solana/web3.js");
const clusterApi = process.env.SOLANA_CLUSTER;
module.exports = {
getConfirmedSignaturesForAddress: async address => {
try {
const connection = new web3.Connection(web3.clusterApiUrl(clusterApi), "confirmed");
const result = await connection.getSignaturesForAddress(address, {
limit: 25
});
return {
tx: result,
status: true
};
} catch (e) {
return {
status: false,
error: e.message
};
}
}
}
And every time I call this function I get this error:
{ status: false, error: 'address.toBase58 is not a function' }
I was trying to send it already converted to Base58, but it just doesn't work. What's wrong?
This is how I solved this problem. Generally speaking, you need to convert it not just by converting to pure Base58, but like this:
const web3 = require("#solana/web3.js");
const bip39 = require("bip39");
const getKeyFromMemonic = async mnemonic => {
return new Promise((resolve, reject) => {
bip39
.mnemonicToSeed(mnemonic)
.then(buffer => {
const a = new Uint8Array(buffer.toJSON().data.slice(0, 32));
const key = web3.Keypair.fromSeed(a);
resolve(key);
})
.catch(err => reject(err));
});
};
getSignaturesForAddress: async address => {
try {
const key = await getKeyFromMemonic(address);
const connection = new web3.Connection(web3.clusterApiUrl(clusterApi), "confirmed");
const result = await connection.getSignaturesForAddress(key.publicKey);
return {
tx: result,
status: true
};
} catch (e) {
return {
status: false,
error: e.message
};
}
}

Run code after executing promise in Javascript

I am trying to save to json the values returned from indeed api. I use indeed-scraper code from github https://github.com/rynobax/indeed-scraper
My code:
... required files ...
const parsedResults = []
indeed.query(queryOptions).then(response => {
response.forEach((res,i) => {
setTimeout(function(){
let url = res.url
let resultCount = 0
console.log(`\n Scraping of ${url} initiated...\n`)
const getWebsiteContent = async (url) => {
try {
const response = await axios.get(url)
const $ = cheerio.load(response.data)
...get scraped data...
parsedResults.push(metadata)
} catch (error) {
exportResults(parsedResults)
console.error(error)
}
}
getWebsiteContent(url)
}
, i*3000);
});
});
const outputFile = 'data.json'
const fs = require('fs');
const exportResults = (parsedResults) => {
fs.writeFile(outputFile, JSON.stringify(parsedResults, null, 4), (err) => {
if (err) {
console.log(err)
}
console.log(`\n ${parsedResults.length} Results exported successfully to ${outputFile}\n`)
})
}
parsedResults is not accessible in last portion of script, so to save as json file.
Any help appreciated!

Imported async data from another module in Node does not resolve, how can I fix this?

This is the module that collections and exports async data: scraper.js
const express = require('express')
const cheerio = require('cheerio')
const request = require("tinyreq")
const fs = require('fs')
const _ = require('lodash')
const uuid = require('uuid/v4')
const async = require('async')
const mental_models = {
url: 'https://www.farnamstreetblog.com/mental-models/',
data: {}
}
const decision_making = {
url: 'https://www.farnamstreetblog.com/smart-decisions/',
data: {}
}
const cognitive_bias = {
url: 'https://betterhumans.coach.me/cognitive-bias-cheat-sheet-55a472476b18',
data: {}
}
const DATA_URLS = [mental_models, decision_making, cognitive_bias]
const filterScrape = async (source, params) => {
let filtered_data = {
topics: [],
content: [],
additional_content: []
}
let response = await scrape(source)
try {
let $ = cheerio.load(response)
params.forEach((elem) => {
let headers = ['h1', 'h2', 'h3']
if ($(elem) && headers.includes(elem)) {
let topic = {}
let content = {}
let id = uuid()
topic.id = id
topic.text = $(elem).text()
if ($(elem).closest('p')) {
content.text = $(elem).closest('p').text()
content.id = id
}
filtered_data.topics.push(topic)
filtered_data.content.push(content)
} else if ($(elem) && !headers.includes(elem)) {
let content = {}
let id = uuid()
content.text = $(elem).text()
content.id = id
filtered_data.additional_content.push(content)
} else {
}
})
}
catch (err) {
console.log(err)
}
return filtered_data
}
const scrape = (source) => {
return new Promise((resolve, reject) => {
request(source.url, function (err, body) {
if (err) {
reject(err)
return
}
resolve(body)
})
})
}
const DATA = _.map(DATA_URLS, async (source) => {
let params = ['h1', 'h2', 'h3', 'p']
let new_data = await filterScrape(source, params)
try {
source.data = new_data
}
catch (err) {
console.log(err)
}
})
module.exports = DATA
This is the module that imports the data: neural.js
const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const DATA = require('./scraper')
console.log(DATA)
Obviously not much going on, I've removed the code since the variable doesn't resolve. When logged it logs a promise but the promise does not resolve. However in the imported module, the promise is logged and then resolves. What gives? Should I import a function that resolves the data?
Of course it would be best to import that function, however it won't change the issue in your code which is here:
const DATA = _.map(DATA_URLS, async (source) => {
Lodash doesn't support async iteration - so you need to have some other method, one would be to use the newest nodejs version (10.x) and make use of async iteration - but that won't use the full power of asynchronous code.
You can also use scramjet - a framework my company is supporting. The code above would take the following form:
const {DataStream} = require("scramjet");
const DATA_URLS = [mental_models, decision_making, cognitive_bias];
module.exports = async () => DataStream.fromArray(DATA_URLS)
.setOptions({maxParallel: 2}) // if you need to limit that at all.
.map(async ({url}) => {
let params = ['h1', 'h2', 'h3', 'p']
let data = await filterScrape(source, params);
return { url, data };
})
.toArray();
The other file would take the following form:
const brain = require('brain')
const neural_net = new brain.NeuralNetwork()
const scraper = require('./scraper')
(async (){
const DATA = await scraper();
console.log(DATA); // or do whatever else you were expecting...
})();

Categories