I have a Redis server with 2 keys. I'm trying to fetch both the keys and combine them into 1 response. I'm using node-redis.
var response = {};
RedisClient.keys('*', function (err, keys) { // gets all keys
async.each( // asynchronously maps over them and parses the values
keys,
function (err, key) {
RedisClient.get(key, function (err, value) {
if (err) throw err;
responsekey] = JSON.parse(value);
});
},
function () {
res.send(response); // finally sends response object
}
);
});
This gets all keys, maps over them asynchronously, and finally parses the values and sends them. The problem is that the redis docs and every resource I've read has said not to use keys. Is there another way to do this in node using SCAN?
Related
I have a CSV file that contains 21k records(1 word alphanumeric type/line). I need to read these records and send them to an API in JSON key-value pair format for some processing that accepts only 500 elements at a time. I have a solution on my mind but I wanted to know that is there a better or more efficient solution/Algorithm for this?
Algorithm:
Load the CSV into an array
Split this 1D array into N array with fix length of 500 columns(elements)
With each of these N number of 500 element Array, prepare JSON payload and send to API.
Code:
var dataArray = [];
fs.readFile(inputPath, 'utf8', function (err, data) {
dataArray = data.split(/\r?\n/);
})
var temp = [];
for(i=0;i<dataArray.length;){
temp=[];
for(j=0;(j<500 && i<dataArray.length);j++){
temp.push(data[i]);
i++;
}
// make API call with current values of temp array
makeCallToAPI(temp);
}
I'd use lodash or underscore _.chunk(). Also note that both the fs and API are better handled async.
const _ = require('lodash');
async function callApi(chunk) {
// return a promise that resolves with the result of the api
}
async function readFS(inputPath) {
return new Promise((resolve, reject) => {
fs.readFile(inputPath, 'utf8', function (err, data) {
if (err) reject(err);
else resolve(data.split(/\r?\n/));
});
});
}
async function doTheWork(inputPath) {
const data = await readFS(inputPath);
const chunks = _.chunk(data, 500)
const promises = chunks.map(callApi)
return _.flatten(Promise.all(promises));
}
Also note the use of _.flatten(), since the last Promise.all() will resolve to an array of arrays of chunks of promises.
I am creating a node API using javascript. I have used redis as my key value store.
I created a redis-client in my app and am able to get values for perticular key.
I want to retrieve all keys along with their values.
So Far I have done this :
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
for(var i=0;i<keys.length;i++){
client.get(keys[i], function (error, value) {
if (err) return console.log(err);
var job = {};
job['jobId']=keys[i];
job['data']=value;
jobs.push(job);
});
}
console.log(jobs);
res.json({data:jobs});
}
});
});
but I always get blank array in response.
is there any way to do this in javascript?
Thanks
First of all, the issue in your question is that, inside the for loop, client.get is invoked with an asynchronous callback where the synchronous for loop will not wait for the asynchronous callback and hence the next line res.json({data:jobs}); is getting called immediately after the for loop before the asynchronous callbacks. At the time of the line res.json({data:jobs}); is getting invoked, the array jobs is still empty [] and getting returned with the response.
To mitigate this, you should use any promise modules like async, bluebird, ES6 Promise etc.
Modified code using async module,
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
async.map(keys, function(key, cb) {
client.get(key, function (error, value) {
if (error) return cb(error);
var job = {};
job['jobId']=key;
job['data']=value;
cb(null, job);
});
}, function (error, results) {
if (error) return console.log(error);
console.log(results);
res.json({data:results});
});
}
});
});
But from the Redis documentation, it is observed that usage of
Keys are intended for debugging and special operations, such as
changing your keyspace layout and not advisable to production
environments.
Hence, I would suggest using another module called redisscan as below which uses SCAN instead of KEYS as suggested in the Redis documentation.
Something like,
var redisScan = require('redisscan');
var redis = require('redis').createClient();
redisScan({
redis: redis,
each_callback: function (type, key, subkey, value, cb) {
console.log(type, key, subkey, value);
cb();
},
done_callback: function (err) {
console.log("-=-=-=-=-=--=-=-=-");
redis.quit();
}
});
Combination of 2 requests:
import * as ioredis from 'ioredis';
const redis = new ioredis({
port: redisPort,
host: redisServer,
password: '',
db: 0
});
const keys = await redis.collection.keys('*');
const values = await redis.collection.mget(keys);
Order will be the same for both arrays.
This will get all keys but with no values:
const redis = require('redis');
const client = redis.createClient();
client.keys('*', (err, keys) => {
// ...
});
Now you need to get the values for those keys in a usual way. For example:
Promise.all(keys.map(key => client.getAsync(key))).then(values => {
// ...
});
or with async module or in any way you like.
You should never do this. First off, it is not recommended to use KEYS * in production. Second, this does not scale (cluster).
You can organise your cached entries into SETs and query for the items within the SET, then retrieve the references keys. This also makes invalidation easier.
Have a look at some data storage best practices.
https://redis.io/topics/data-types-intro
how to get all keys and values in redis in javascript?
You may find something useful in this link
https://github.com/NodeRedis/node_redis/tree/master/examples
I've spent so much time trying to find the answer to this on here and come up with nothing. Hoping someone can enlighten me..
I have code which is making an async call to a database and returning data in a callback function (in my case I'm using MongoClient, which returns a Promise). However, I can't work out how to use the resulting data to actually set function-level variables - whenever I try to do it the resulting value that I log is either undefined or a pending Promise object.
There's lots of posts on this subject but I can't find any methods that work when I try to apply them. Any and all help gratefully received!
function lookupOneDbEntry(key, value) {
var responseData = "initial data"
// search for the entry in the database
db.collection("my_collection").findOne({key: value}, function(err, result) {
if (err) {
//if database throws an error
responseData = "db error";
}
else {
// if the entry is found, return the data
responseData = result;
}
});
return responseData;
}
EDIT: I am aware of other posts on this (like this one here and, while exhaustive documentation is useful to an extent, I;m having trouble using this information practically in a real-life implementation like the one above. Hence my question here.
Async calls happens outside of the call stack that you are on. you can't return it onto the current stack.
So we use the promises to hook into the results of our call.
function lookupOneDbEntry(key, value) {
return new Promise(function (resolve, reject) {
// search for the entry in the database
db.collection("my_collection").findOne({key: value}, function(err, result) {
if (err) {
//if database throws an error
reject(err);
}
else {
// if the entry is found, return the data
resolve(result);
}
});
});
}
lockupOneDbEntry('myKey', 'myValue').then(function (result) {
console.log('result', result);
}, function (err) {
console.log("error!", err);
});
After a long while of experimenting I've finally managed to do it - I didn't need any fancy callbacks or additional Promises in the end, I just removed the optional callback in the database request and instead processed the returned promise separately.
function lookupOneDbEntry(key, value) {
var responseData = "initial data";
var solution = db.collection("accounting_module").findOne({key: value});
solution.then(function (result) {
// validation of result here
responseData = result;
});
return responseData;
}
So, we are trying to rewrite our express server into Rx. It is currently using async for all stream operations. The code looks like the following:
var async = require('async');
function getCountAndChannels(name, cb){
var tasks = [
function(cb) {
//does a mongoDB search and returns count
},
function(cb) {
//does a findOne mongoDB search and returns
}
];
async.parallel(tasks, cb);
}
router.get('data', function(req, res) { //router is the express router
var recorders = req.query.recorders.split(',');
async.map(recorders, function(name, cb) {
getCountAndChannels(name, cb);
}, function(err, countsAndChannels) {
if(err) throw err;
// here countsAndChannels is an array with first element the count
// and second element the document.
// do other async stuff based on the results
res.status(200).json('send some calculations');
});
The thing here I have to do is loop over the array of recorders and for each one calculate the two mongoDB searches. I have tried using Rx.Observable.merge which doesn't return the results in an array but in 2 different calls of the callback. So, then I tried Rx.Observable.zip which I believe is what I'm looking for.
The problem is I don't know how to loop over the recorders and send the result when all operations are finished. Because a simple forEach loop will throw a Cannot set headers after they are sent error.
This is what I have so far:
recorders.forEach(recorder => {
Rx.Observable.zip([
search1,
search2
]).subscribe(
(countsAndChannels) => {
// do stuff
res.send('the results');
},
err => res.status(500).json(err),
() => res.send('OK')
);
});
Haven't used Rx before, so any help is appreciated.
It might be easier to convert your list of recorders to an Observable stream, then flatMap over each recorder (ie perform your async processing), then call toArray to store all the results into an array:
var recorder$ = Rx.Observable.from(recorders);
var countsAndChannels$ = recorder$
.flatMap(performAsyncTask);
// allResults$ will emit once all of the async work is complete
var allResults$= countsAndChannels$.toArray();
allResults$.subscribe(results => {
// Send response to client;
});
I'm trying to execute several async methods of redis with the following code
var redis = require("redis");
var client = redis.createClient();
var async = require("asyncjs");
async.list([
client.hincrby("traffic:" + siteId, 'x', 1),
client.hincrby("traffic:" + siteId, 'y', 1),
client.hincrby("traffic:" + siteId, 'z', 1)
]).call().end(function(err, result)
{
console.log(err); // returns error [TypeError: Object true has no method 'apply']
console.log(result); // undefined
if(err) return false;
return result;
});
All the methods execute successfully
but i get the error [TypeError: Object true has no method 'apply']
the method is executed and returns true, and its probably interpreting that as true, but i don't see why it has to use the method apply on it?
I can get the result of the increment by adding a function(err, result) to client.hincrby as last element.. but how do i get all the results in the result variable in the end function?
I suppose the asyncjs module you use is the one documented at:
https://github.com/fjakobs/async.js
In your code:
list() is a generator. It allows the array to be iterated by asyncjs. The array is an array of values.
call() is a mapper which calls each item. The items has therefore to be callable (i.e. they have to be callbacks).
end() is a termination end point, called when the iteration is over. As a parameter, you only get the last value of the sequence (not the whole sequence).
You got the "[TypeError: Object true has no method 'apply']" error because the list you have built is not a list of callbacks. It is a list of values.
Here is some code which should do what you want:
var redis = require("redis");
var client = redis.createClient();
var async = require("asyncjs");
function main() {
var siteId = 1;
async
.list(['x','y','z'])
.map( function (item,next) {
client.hincrby('traffic:' + siteId, item, 1, function (err,res) {
next(err,res)
})
})
.toArray( function(err,res) {
console.log(err);
console.log(res);
});
}
main()
Please note here we use map() instead of call(), and toArray() instead of end().