Related
In Node.js I have to read files in a folder and for each file get file handler info, this is my simplest implementation using fs.readdir:
FileServer.prototype.listLocal = function (params) {
var self = this;
var options = {
limit: 100,
desc: 1
};
// override defaults
for (var attrname in params) { options[attrname] = params[attrname]; }
// media path is the media folder
var mediaDir = path.join(self._options.mediaDir, path.sep);
return new Promise((resolve, reject) => {
fs.readdir(mediaDir, (error, results) => {
if (error) {
self.logger.error("FileServer.list error:%s", error);
return reject(error);
} else { // list files
// cut to max files
results = results.slice(0, options.limit);
// filter default ext
results = results.filter(item => {
return (item.indexOf('.mp3') > -1);
});
// format meta data
results = results.map(file => {
var filePath = path.join(self._options.mediaDir, path.sep, file);
var item = {
name: file,
path: filePath
};
const fd = fs.openSync(filePath, 'r');
var fstat = fs.fstatSync(fd);
// file size in bytes
item.size = fstat.size;
item.sizehr = self.formatSizeUnits(fstat.size);
// "Birth Time" Time of file creation. Set once when the file is created.
item.birthtime = fstat.birthtime;
// "Modified Time" Time when file data last modified.
item.mtime = fstat.mtime;
// "Access Time" Time when file data last accessed.
item.atime = fstat.atime;
item.timestamp = new Date(item.mtime).getTime();
item.media_id = path.basename(filePath, '.mp3');
fs.closeSync(fd);//close file
return item;
});
if (options.desc) { // sort by most recent
results.sort(function (a, b) {
return b.timestamp - a.timestamp;
});
} else { // sort by older
results.sort(function (a, b) {
return a.timestamp - b.timestamp;
});
}
return resolve(results);
}
})
});
}
so that for each file I get an array of items
{
"name": "sample121.mp3",
"path": "/data/sample121.mp3",
"size": 5751405,
"sizehr": "5.4850 MB",
"birthtime": "2018-10-08T15:26:08.397Z",
"mtime": "2018-10-08T15:26:11.650Z",
"atime": "2018-10-10T09:01:48.534Z",
"timestamp": 1539012371650,
"media_id": "sample121"
}
That said, the problem is it's knonw that node.js fs.readdir may freeze Node I/O Loop when the folder to list has a large number of files, let's say from ten thousands to hundred thousands and more.
This is a known issue - see here for more info.
There are also plans to improve fs.readdir in a some way, like streaming - see here about this.
In the meanwhile I'm searching for like a patch to this, because my folders are pretty large.
Since the problem is the Event Loop get frozen, someone proposed a solution using process.nextTick, that I have ensembled here
FileServer.prototype.listLocalNextTick = function (params) {
var self = this;
var options = {
limit: 100,
desc: 1
};
// override defaults
for (var attrname in params) { options[attrname] = params[attrname]; }
// media path is the media folder
var mediaDir = path.join(self._options.mediaDir, path.sep);
return new Promise((resolve, reject) => {
var AsyncArrayProcessor = function (inArray, inEntryProcessingFunction) {
var elemNum = 0;
var arrLen = inArray.length;
var ArrayIterator = function () {
inEntryProcessingFunction(inArray[elemNum]);
elemNum++;
if (elemNum < arrLen) process.nextTick(ArrayIterator);
}
if (elemNum < arrLen) process.nextTick(ArrayIterator);
}
fs.readdir(mediaDir, function (error, results) {
if (error) {
self.logger.error("FileServer.list error:%s", error);
return reject(error);
}
// cut to max files
results = results.slice(0, options.limit);
// filter default ext
results = results.filter(item => {
return (item.indexOf('.mp3') > -1);
});
var ProcessDirectoryEntry = function (file) {
// This may be as complex as you may fit in a single event loop
var filePath = path.join(self._options.mediaDir, path.sep, file);
var item = {
name: file,
path: filePath
};
const fd = fs.openSync(filePath, 'r');
var fstat = fs.fstatSync(fd);
// file size in bytes
item.size = fstat.size;
item.sizehr = self.formatSizeUnits(fstat.size);
// "Birth Time" Time of file creation. Set once when the file is created.
item.birthtime = fstat.birthtime;
// "Modified Time" Time when file data last modified.
item.mtime = fstat.mtime;
// "Access Time" Time when file data last accessed.
item.atime = fstat.atime;
item.timestamp = new Date(item.mtime).getTime();
item.media_id = path.basename(filePath, '.mp3');
// map to file item
file = item;
}//ProcessDirectoryEntry
// LP: fs.readdir() callback is finished, event loop continues...
AsyncArrayProcessor(results, ProcessDirectoryEntry);
if (options.desc) { // sort by most recent
results.sort(function (a, b) {
return b.timestamp - a.timestamp;
});
} else { // sort by older
results.sort(function (a, b) {
return a.timestamp - b.timestamp;
});
}
return resolve(results);
});
});
}//listLocalNextTick
This seems to avoid the original issue, but I cannot anymore map the files lists to the items with file handler I did before, because when running the AsyncArrayProcessor on the files list, thus the ProcessDirectoryEntry on each file entry the async nature of process.nextTick causes that I cannot get back the results array modified as in the previous listLocal function where I just did an iterative array.map of the results array.
How to patch the listLocalNextTick to behave like the listLocal but keeping process.nextTick approach?
[UPDATE]
According to the proposed solution, this is the best implementation so far:
/**
* Scan files in directory
* #param {String} needle
* #param {object} options
* #returns {nodeStream}
*/
scanDirStream : function(needle,params) {
var options = {
type: 'f',
name: '*'
};
for (var attrname in params) { options[attrname] = params[attrname]; }
return new Promise((resolve, reject) => {
var opt=[needle];
for (var k in options) {
var v = options[k];
if (!Util.empty(v)) {
opt.push('-' + k);
opt.push(v);
}
};
var data='';
var listing = spawn('find',opt)
listing.stdout.on('data', _data => {
var buff=Buffer.from(_data, 'utf-8').toString();
if(buff!='') data+=buff;
})
listing.stderr.on('data', error => {
return reject(Buffer.from(error, 'utf-8').toString());
});
listing.on('close', (code) => {
var res = data.split('\n');
return resolve(res);
});
});
Example of usage:
scanDirStream(mediaRoot,{
name: '*.mp3'
})
.then(results => {
console.info("files:%d", results);
})
.catch(error => {
console.error("error %s", error);
});
This can be eventually modified to add a tick callback at every stdout.on event emitted when getting a new file in the directory listening.
I have Created a wrapper around find for it but you could use dir or ls in the same way.
const { spawn } = require('child_process');
/**
* findNodeStream
* #param {String} dir
* #returns {nodeStream}
*/
const findNodeStream = (dir,options) => spawn('find',[dir,options].flat().filter(x=>x));
/**
* Usage Example:
let listing = findNodeStream('dir',[options])
listing.stdout.on('data', d=>console.log(d.toString()))
listing.stderr.on('data', d=>console.log(d.toString()))
listing.on('close', (code) => {
console.log(`child process exited with code ${code}`);
});
*/
this allows you to stream a directory chunked and not in a whole as fs.readdir does.
Important
NodeJS > 12.11.1 will have async readdir support
Landed in cbd8d71 ( https://github.com/nodejs/node/commit/cbd8d715b2286e5726e6988921f5c870cbf74127 ) as fs{Promises}.opendir(), which returns an fs.Dir, which exposes an async iterator. tada
https://nodejs.org/api/fs.html#fs_fspromises_opendir_path_options
const fs = require('fs');
async function print(path) {
const dir = await fs.promises.opendir(path);
for await (const dirent of dir) {
console.log(dirent.name);
}
}
print('./').catch(console.error);
I need to remove all documents from my mongo db, which dont exists in new array with objects.
So I have array with objects like :
var items = [
{product_id:15, pr_name: 'a', description : 'desc'},
{product_id:44, pr_name: 'b', description : 'desc2'}
{product_id:32, pr_name: 'c', description : 'desc3'}];
and I have array with db values which I get by calling Model.find({}).
So now I do it in a 'straight' way:
async.each(products, function (dbProduct, callback) { //cycle for products removing
var equals = false;
async.each(items, function(product, callback){
if (dbProduct.product_id === product.product_id){
product.description = dbProduct.description;// I need to save desc from db product to new product
equals = true;
}
callback();
});
if (!equals) {
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
}
});
But its blocks the whole app and its very slow, because I have around 5000 values in my items array and in database too. So its very huge cycle numbers.
Maybe there can be a faster way?
UPDATE1
Using code below, from TbWill4321 answer:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Its takes around 11 sec(blocks whole web-app) and takes 12 362 878 cycles for me.
So maybe somebody can advise me something?
The Async library does not execute synchronous code in an asynchronous fashion.
5000 items is not a huge number for JavaScript, as I've worked on Big Data set's with 5 million+ points and it doesn't take long. You can get better performance by structuring like this:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Among the many problems you may have, off the top of my head you may want to start off by changing this bit:
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
Being within a .each() call, you'll make one call to the database for each element you want to delete. It's better to store all the ids in one array and then make a single query to delete all elements that have an _id that is in that array. Like this
Product.remove({ _id: {$in: myArrayWithIds} }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
On another note, since async will execute synchronously, node.js does offer setImmediate() (docs here), that will execute the function from within the event loop. So basically you can "pause" execution of new elements and serve any incoming requests to simulate "non-blocking" processing.
I'm trying to build a small wrapper library for the node redis module.
var redis = require('redis'),
client = redis.createClient();
module.exports = {
/*
* Time To Live
*
* Time in seconds the cache will remain in
* memory.
*/
ttl: 120,
/*
* Time To Refresh
*
* Time buffer in seconds when the cache should
* be refreshed. On a cache check, if the ttl
* is less than the ttr, the cache will be
* refreshed.
*/
ttr: 60,
/*
* Check Cache
*
* Middleware to check if the request is cached
* before executing the full request.
*/
check: function (req, res, next) {
var key = req.url.slice(1, req.url.length).replace('/', ':');
client.get(key, function (err, value) {
if (value !== null) {
res.send(value);
client.ttl(key, function (err, ttl) {
if (ttl < this.ttr) {
return next();
}
return;
});
} else {
return next();
}
})
},
/*
* Set Cache
*
* Takes a key and a value and stores it in redis.
* Also takes the full response object "res" and
* handles sending the response if it has not
* already been sent.
*/
set: function (url, value, res) {
var key = url.slice(1, url.length).replace('/', ':');
client.set(key, value);
client.expire(key, this.ttl);
if (!res.headersSent) {
res.send(value);
}
return;
},
/*
* Keygen Cache
*
* Takes a urls substring and creates a keyname
* in line with redis best practices
*/
keygen: function (url) {
var key = url.slice(0,1).replace('/', ':');
console.log(key);
return key;
}
};
I can't figure out how to use the 'this' keyword properly. If I try to reference this.ttl or this.ttr or call this.keygen from another function within the module.exports object it always turns up undefined. What is a structure I should be using to enable referencing functions internal to the same object?
You have nested functions in your check function, so the this-keyword relates to the inner function context and not to your object.
First solution:
function (req, res, next) {
var _this = this;
var key = req.url.slice(1, req.url.length).replace('/', ':');
client.get(key, function (err, value) {
if (value !== null) {
res.send(value);
client.ttl(key, function (err, ttl) {
if (ttl < _this.ttr) {
return next();
}
return;
});
} else {
return next();
}
})
}
And if you are using the function as a callback function bind your object to the callback function like this:
// replace "yourModule" with the name of your module
var yourModule = require('./yourModule.js');
// an example for expressjs (use get, post ... or something else)
app.get('/some/url', yourModule.check.bind(yourModule));
Second solution (IMHO the better one):
Another solution is to use local variables in your module (these are only visible in your module -> and this is what you want).
This is way more easier:
/*
* Time To Live
*
* Time in seconds the cache will remain in
* memory.
*/
var ttl = 120;
/*
* Time To Refresh
*
* Time buffer in seconds when the cache should
* be refreshed. On a cache check, if the ttl
* is less than the ttr, the cache will be
* refreshed.
*/
var ttr = 60;
/*
* Check Cache
*
* Middleware to check if the request is cached
* before executing the full request.
*/
var check = function (req, res, next) {
var key = req.url.slice(1, req.url.length).replace('/', ':');
client.get(key, function (err, value) {
if (value !== null) {
res.send(value);
client.ttl(key, function (err, ttl) {
if (ttl < ttr) {
return next();
}
return;
});
} else {
return next();
}
})
};
/*
* Set Cache
*
* Takes a key and a value and stores it in redis.
* Also takes the full response object "res" and
* handles sending the response if it has not
* already been sent.
*/
var set = function (url, value, res) {
var key = url.slice(1, url.length).replace('/', ':');
client.set(key, value);
client.expire(key, ttl);
if (!res.headersSent) {
res.send(value);
}
return;
};
/*
* Keygen Cache
*
* Takes a urls substring and creates a keyname
* in line with redis best practices
*/
var keygen = function (url) {
var key = url.slice(0,1).replace('/', ':');
console.log(key);
return key;
};
/**
* PUBLIC API
*/
module.exports = {
check: check,
set: set,
keygen: keygen
};
Now because you are not relying on the context of the module you can simply do this:
// replace "yourModule" with the name of your module
var yourModule = require('./yourModule.js');
// an example for expressjs (use get, post ... or something else)
app.get('/some/url', yourModule.check);
Is there any option to perform bulk upserts with mongoose? So basically having an array and insert each element if it not exists or update it if it exists? (I am using customs _ids)
When I do use .insert MongoDB returns an error E11000 for duplicate keys (which should be updated). Inserting multiple new document works fine though:
var Users = self.db.collection('Users');
Users.insert(data, function(err){
if (err) {
callback(err);
}
else {
callback(null);
}
});
Using .save returns an error that the parameter must be a single document:
Users.save(data, function(err){
...
}
This answer suggest there is no such option, however it is specific for C# and also already 3 years old. So I was wondering if there is any option to do that using mongoose?
Thank you!
Not in "mongoose" specifically, or at least not yet as of writing. The MongoDB shell as of the 2.6 release actually uses the "Bulk operations API" "under the hood" as it were for all of the general helper methods. In it's implementation, it tries to do this first, and if an older version server is detected then there is a "fallback" to the legacy implementation.
All of the mongoose methods "currently" use the "legacy" implementation or the write concern response and the basic legacy methods. But there is a .collection accessor from any given mongoose model that essentially accesses the "collection object" from the underlying "node native driver" on which mongoose is implemented itself:
var mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/test');
var sampleSchema = new Schema({},{ "strict": false });
var Sample = mongoose.model( "Sample", sampleSchema, "sample" );
mongoose.connection.on("open", function(err,conn) {
var bulk = Sample.collection.initializeOrderedBulkOp();
var counter = 0;
// representing a long loop
for ( var x = 0; x < 100000; x++ ) {
bulk.find(/* some search */).upsert().updateOne(
/* update conditions */
});
counter++;
if ( counter % 1000 == 0 )
bulk.execute(function(err,result) {
bulk = Sample.collection.initializeOrderedBulkOp();
});
}
if ( counter % 1000 != 0 )
bulk.execute(function(err,result) {
// maybe do something with result
});
});
The main catch there being that "mongoose methods" are actually aware that a connection may not actually be made yet and "queue" until this is complete. The native driver you are "digging into" does not make this distinction.
So you really have to be aware that the connection is established in some way or form. But you can use the native driver methods as long as you are careful with what you are doing.
You don't need to manage limit (1000) as #neil-lunn suggested. Mongoose does this already. I used his great answer as a basis for this complete Promise-based implementation & example:
var Promise = require('bluebird');
var mongoose = require('mongoose');
var Show = mongoose.model('Show', {
"id": Number,
"title": String,
"provider": {'type':String, 'default':'eztv'}
});
/**
* Atomic connect Promise - not sure if I need this, might be in mongoose already..
* #return {Priomise}
*/
function connect(uri, options){
return new Promise(function(resolve, reject){
mongoose.connect(uri, options, function(err){
if (err) return reject(err);
resolve(mongoose.connection);
});
});
}
/**
* Bulk-upsert an array of records
* #param {Array} records List of records to update
* #param {Model} Model Mongoose model to update
* #param {Object} match Database field to match
* #return {Promise} always resolves a BulkWriteResult
*/
function save(records, Model, match){
match = match || 'id';
return new Promise(function(resolve, reject){
var bulk = Model.collection.initializeUnorderedBulkOp();
records.forEach(function(record){
var query = {};
query[match] = record[match];
bulk.find(query).upsert().updateOne( record );
});
bulk.execute(function(err, bulkres){
if (err) return reject(err);
resolve(bulkres);
});
});
}
/**
* Map function for EZTV-to-Show
* #param {Object} show EZTV show
* #return {Object} Mongoose Show object
*/
function mapEZ(show){
return {
title: show.title,
id: Number(show.id),
provider: 'eztv'
};
}
// if you are not using EZTV, put shows in here
var shows = []; // giant array of {id: X, title: "X"}
// var eztv = require('eztv');
// eztv.getShows({}, function(err, shows){
// if(err) return console.log('EZ Error:', err);
// var shows = shows.map(mapEZ);
console.log('found', shows.length, 'shows.');
connect('mongodb://localhost/tv', {}).then(function(db){
save(shows, Show).then(function(bulkRes){
console.log('Bulk complete.', bulkRes);
db.close();
}, function(err){
console.log('Bulk Error:', err);
db.close();
});
}, function(err){
console.log('DB Error:', err);
});
// });
This has the bonus of closing the connection when it's done, displaying any errors if you care, but ignoring them if not (error callbacks in Promises are optional.) It's also very fast. Just leaving this here to share my findings. You can uncomment the eztv stuff if you want to save all eztv shows to a database, as an example.
await Model.bulkWrite(docs.map(doc => ({
updateOne: {
filter: {id: doc.id},
update: doc,
upsert: true
}
})))
Or more verbose:
const bulkOps = docs.map(doc => ({
updateOne: {
filter: {id: doc.id},
update: doc,
upsert: true
}
}))
Model.bulkWrite(bulkOps)
.then(bulkWriteOpResult => console.log('BULK update OK:', bulkWriteOpResult))
.catch(err => console.error('BULK update error:', err))
https://stackoverflow.com/a/60330161/5318303
I have released a plugin for Mongoose that exposes a static upsertMany method to perform bulk upsert operations with a promise interface.
An added benefit of using this plugin over initializing your own bulk op on the underlying collection, is that this plugin converts your data to Mongoose model's first, and then back to plain objects before the upsert. This ensures Mongoose schema validation is applied, and data is depopulated and fit for raw insertion.
https://github.com/meanie/mongoose-upsert-many
https://www.npmjs.com/package/#meanie/mongoose-upsert-many
Hope it helps!
If you're not seeing the bulk methods in your db.collection ie you're getting a error to the effect of
xxx variable has no method: initializeOrderedBulkOp()
Try updating your mongoose version. Apparently older mongoose versions don't pass through all of the underlying mongo db.collection methods.
npm install mongoose
took care of it for me.
I had to achieve this recently while storing products in my ecommerce app. My database used to timeout as I had to upsert 10000 items every 4 hours. One option for me was to set the socketTimeoutMS and connectTimeoutMS in mongoose while connecting to the database but it sorta felt hacky and I did not want to manipulate connection timeout defaults of the database. I also see that the solution by #neil lunn takes a simple sync approach of taking a modulus inside the for loop. Here is an async version of mine that I believe does the job much better
let BATCH_SIZE = 500
Array.prototype.chunk = function (groupsize) {
var sets = [];
var chunks = this.length / groupsize;
for (var i = 0, j = 0; i < chunks; i++ , j += groupsize) {
sets[i] = this.slice(j, j + groupsize);
}
return sets;
}
function upsertDiscountedProducts(products) {
//Take the input array of products and divide it into chunks of BATCH_SIZE
let chunks = products.chunk(BATCH_SIZE), current = 0
console.log('Number of chunks ', chunks.length)
let bulk = models.Product.collection.initializeUnorderedBulkOp();
//Get the current time as timestamp
let timestamp = new Date(),
//Keep track of the number of items being looped
pendingCount = 0,
inserted = 0,
upserted = 0,
matched = 0,
modified = 0,
removed = 0,
//If atleast one upsert was performed
upsertHappened = false;
//Call the load function to get started
load()
function load() {
//If we have a chunk to process
if (current < chunks.length) {
console.log('Current value ', current)
for (let i = 0; i < chunks[current].length; i++) {
//For each item set the updated timestamp to the current time
let item = chunks[current][i]
//Set the updated timestamp on each item
item.updatedAt = timestamp;
bulk.find({ _id: item._id })
.upsert()
.updateOne({
"$set": item,
//If the item is being newly inserted, set a created timestamp on it
"$setOnInsert": {
"createdAt": timestamp
}
})
}
//Execute the bulk operation for the current chunk
bulk.execute((error, result) => {
if (error) {
console.error('Error while inserting products' + JSON.stringify(error))
next()
}
else {
//Atleast one upsert has happened
upsertHappened = true;
inserted += result.nInserted
upserted += result.nUpserted
matched += result.nMatched
modified += result.nModified
removed += result.nRemoved
//Move to the next chunk
next()
}
})
}
else {
console.log("Calling finish")
finish()
}
}
function next() {
current++;
//Reassign bulk to a new object and call load once again on the new object after incrementing chunk
bulk = models.Product.collection.initializeUnorderedBulkOp();
setTimeout(load, 0)
}
function finish() {
console.log('Inserted ', inserted + ' Upserted ', upserted, ' Matched ', matched, ' Modified ', modified, ' Removed ', removed)
//If atleast one chunk was inserted, remove all items with a 0% discount or not updated in the latest upsert
if (upsertHappened) {
console.log("Calling remove")
remove()
}
}
/**
* Remove all the items that were not updated in the recent upsert or those items with a discount of 0
*/
function remove() {
models.Product.remove(
{
"$or":
[{
"updatedAt": { "$lt": timestamp }
},
{
"discount": { "$eq": 0 }
}]
}, (error, obj) => {
if (error) {
console.log('Error while removing', JSON.stringify(error))
}
else {
if (obj.result.n === 0) {
console.log('Nothing was removed')
} else {
console.log('Removed ' + obj.result.n + ' documents')
}
}
}
)
}
}
You can use mongoose's Model.bulkWrite()
const res = await Character.bulkWrite([
{
updateOne: {
filter: { name: 'Will Riker' },
update: { age: 29 },
upsert: true
}
},
{
updateOne: {
filter: { name: 'Geordi La Forge' },
update: { age: 29 },
upsert: true
}
}
]);
reference : https://masteringjs.io/tutorials/mongoose/upsert
UPDATE: I found a solution
I had to do exactly as follows:
In the extension dir I created a new dir named "components".
Inside that dir I create a js file with my custom auto-complete name, in this case "simpleautocomplete.js".
The content of that js file is:
/*
* https://developer.mozilla.org/en/How_to_implement_custom_autocomplete_search_component
*/
const Ci = Components.interfaces;
const CLASS_ID = Components.ID("6224daa1-71a2-4d1a-ad90-01ca1c08e323");
const CLASS_NAME = "Simple AutoComplete";
const CONTRACT_ID = "#mozilla.org/autocomplete/search;1?name=simple-autocomplete";
try{
Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
} catch (x) {
}
// Implements nsIAutoCompleteResult
function SimpleAutoCompleteResult(searchString, searchResult,
defaultIndex, errorDescription,
results, comments) {
this._searchString = searchString;
this._searchResult = searchResult;
this._defaultIndex = defaultIndex;
this._errorDescription = errorDescription;
this._results = results;
this._comments = comments;
}
SimpleAutoCompleteResult.prototype = {
_searchString: "",
_searchResult: 0,
_defaultIndex: 0,
_errorDescription: "",
_results: [],
_comments: [],
getLabelAt: function(index) { return this._results[index]; },
/**
* The original search string
*/
get searchString() {
return this._searchString;
},
/**
* The result code of this result object, either:
* RESULT_IGNORED (invalid searchString)
* RESULT_FAILURE (failure)
* RESULT_NOMATCH (no matches found)
* RESULT_SUCCESS (matches found)
*/
get searchResult() {
return this._searchResult;
},
/**
* Index of the default item that should be entered if none is selected
*/
get defaultIndex() {
return this._defaultIndex;
},
/**
* A string describing the cause of a search failure
*/
get errorDescription() {
return this._errorDescription;
},
/**
* The number of matches
*/
get matchCount() {
return this._results.length;
},
/**
* Get the value of the result at the given index
*/
getValueAt: function(index) {
return this._results[index];
},
/**
* Get the comment of the result at the given index
*/
getCommentAt: function(index) {
return this._comments[index];
},
/**
* Get the style hint for the result at the given index
*/
getStyleAt: function(index) {
if (!this._comments[index])
return null; // not a category label, so no special styling
if (index == 0)
return "suggestfirst"; // category label on first line of results
return "suggesthint"; // category label on any other line of results
},
/**
* Get the image for the result at the given index
* The return value is expected to be an URI to the image to display
*/
getImageAt : function (index) {
return "";
},
/**
* Remove the value at the given index from the autocomplete results.
* If removeFromDb is set to true, the value should be removed from
* persistent storage as well.
*/
removeValueAt: function(index, removeFromDb) {
this._results.splice(index, 1);
this._comments.splice(index, 1);
},
QueryInterface: function(aIID) {
if (!aIID.equals(Ci.nsIAutoCompleteResult) && !aIID.equals(Ci.nsISupports))
throw Components.results.NS_ERROR_NO_INTERFACE;
return this;
}
};
// Implements nsIAutoCompleteSearch
function SimpleAutoCompleteSearch() {
}
SimpleAutoCompleteSearch.prototype = {
classID: CLASS_ID,
classDescription: CLASS_NAME,
contractID: CONTRACT_ID,
/*
* Search for a given string and notify a listener (either synchronously
* or asynchronously) of the result
*
* #param searchString - The string to search for
* #param searchParam - An extra parameter
* #param previousResult - A previous result to use for faster searchinig
* #param listener - A listener to notify when the search is complete
*/
startSearch: function(searchString, searchParam, result, listener) {
// This autocomplete source assumes the developer attached a JSON string
// to the the "autocompletesearchparam" attribute or "searchParam" property
// of the <textbox> element. The JSON is converted into an array and used
// as the source of match data. Any values that match the search string
// are moved into temporary arrays and passed to the AutoCompleteResult
if (searchParam.length > 0) {
var nativeJSON = Components.classes["#mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
var searchResults = nativeJSON.decode(searchParam);
var results = [];
var comments = [];
for (i=0; i<searchResults.length; i++) {
if (searchResults[i].value.indexOf(searchString) == 0) {
results.push(searchResults[i].value);
if (searchResults[i].comment)
comments.push(searchResults[i].comment);
else
comments.push(null);
}
}
var newResult = new SimpleAutoCompleteResult(searchString, Ci.nsIAutoCompleteResult.RESULT_SUCCESS, 0, "", results, comments);
listener.onSearchResult(this, newResult);
}
},
/*
* Stop an asynchronous search that is in progress
*/
stopSearch: function() {
},
QueryInterface: function(aIID) {
if (!aIID.equals(Ci.nsIAutoCompleteSearch) && !aIID.equals(Ci.nsISupports))
throw Components.results.NS_ERROR_NO_INTERFACE;
return this;
},
_QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIAutoCompleteSearch])
};
// Factory
var SimpleAutoCompleteSearchFactory = {
singleton: null,
createInstance: function (aOuter, aIID) {
if (aOuter != null)
throw Components.results.NS_ERROR_NO_AGGREGATION;
if (this.singleton == null)
this.singleton = new SimpleAutoCompleteSearch();
return this.singleton.QueryInterface(aIID);
}
};
// Module
var SimpleAutoCompleteSearchModule = {
registerSelf: function(aCompMgr, aFileSpec, aLocation, aType) {
aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
aCompMgr.registerFactoryLocation(CLASS_ID, CLASS_NAME, CONTRACT_ID, aFileSpec, aLocation, aType);
},
unregisterSelf: function(aCompMgr, aLocation, aType) {
aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
aCompMgr.unregisterFactoryLocation(CLASS_ID, aLocation);
},
getClassObject: function(aCompMgr, aCID, aIID) {
if (!aIID.equals(Components.interfaces.nsIFactory))
throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
if (aCID.equals(CLASS_ID))
return SimpleAutoCompleteSearchFactory;
throw Components.results.NS_ERROR_NO_INTERFACE;
},
canUnload: function(aCompMgr) { return true; }
};
// Module initialization
function NSGetModule(aCompMgr, aFileSpec) { return SimpleAutoCompleteSearchModule; }
if (XPCOMUtils.generateNSGetFactory){
var NSGetFactory = XPCOMUtils.generateNSGetFactory([SimpleAutoCompleteSearch]);
}
I added this code to chrome.manifest:
component {6224daa1-71a2-4d1a-ad90-01ca1c08e323} components/simpleautocomplete.js
contract #mozilla.org/autocomplete/search;1?name=simple-autocomplete {6224daa1-71a2-4d1a-ad90-01ca1c08e323}
In the xul file I added:
<textbox type="autocomplete" autocompletesearch="simple-autocomplete"
autocompletesearchparam='[{"value":"param1"},{"value":"param2"}]' />
important note: when setting the autocompletesearchparam attribute the assignment has to be insind two single quotes and not double quotes. only the values inside has to be in double quotes as in the xul above.
and that's it.
When I tested my extension and typed 'p' in the auto-complete textbox the words "param1" and "param2" poped-up.
I could also set the params in the js file as follows:
var searchTextField = document.getElementById("searchTextField");
var param1 = "Param1", param2 = "Param2";
paramsToSet = "[";
paramsToSet += "{\"value\" : \"" + param1 + "\"},";
paramsToSet += "{\"value\" : \"" + param2 + "\"},";
paramsToSet = paramsToSet.substring(0, paramsToSet.length-1); // to remove the last ","
paramsToSet += "]";
paramsToSet = paramsToSet.toLowerCase(); // important!
searchTextField.setAttribute("autocompletesearchparam", paramsToSet);
note1: notice that the auto-complete textbox only accepts text in small letters.
note2: if setting the params to the textbox is dynamically and taking some time, for example - when the client is typing in the textbox and the typed text is sent to a server so the response containing the params will be attached to the textbox, and this process is taking some time (about a second or half), then the popup auto-complete won't popup because it's autocompletesearchparam was empty when the client started to type. In this case it is possible to force the auto-complete to popup when you have the params to set in the textbox like this:
searchTextField.open = true;
I Found the solution by myself, hope it will helps someone.
I had to do exactly as follows:
In the extension dir I created a new dir named "components".
Inside that dir I create a js file with my custom auto-complete name, in this case "simpleautocomplete.js". The content of that js file is:
/*
* https://developer.mozilla.org/en/How_to_implement_custom_autocomplete_search_component
*/
const Ci = Components.interfaces;
const CLASS_ID = Components.ID("6224daa1-71a2-4d1a-ad90-01ca1c08e323");
const CLASS_NAME = "Simple AutoComplete";
const CONTRACT_ID = "#mozilla.org/autocomplete/search;1?name=simple-autocomplete";
try{
Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
} catch (x) {
}
// Implements nsIAutoCompleteResult
function SimpleAutoCompleteResult(searchString, searchResult,
defaultIndex, errorDescription,
results, comments) {
this._searchString = searchString;
this._searchResult = searchResult;
this._defaultIndex = defaultIndex;
this._errorDescription = errorDescription;
this._results = results;
this._comments = comments;
}
SimpleAutoCompleteResult.prototype = {
_searchString: "",
_searchResult: 0,
_defaultIndex: 0,
_errorDescription: "",
_results: [],
_comments: [],
getLabelAt: function(index) { return this._results[index]; },
/**
* The original search string
*/
get searchString() {
return this._searchString;
},
/**
* The result code of this result object, either:
* RESULT_IGNORED (invalid searchString)
* RESULT_FAILURE (failure)
* RESULT_NOMATCH (no matches found)
* RESULT_SUCCESS (matches found)
*/
get searchResult() {
return this._searchResult;
},
/**
* Index of the default item that should be entered if none is selected
*/
get defaultIndex() {
return this._defaultIndex;
},
/**
* A string describing the cause of a search failure
*/
get errorDescription() {
return this._errorDescription;
},
/**
* The number of matches
*/
get matchCount() {
return this._results.length;
},
/**
* Get the value of the result at the given index
*/
getValueAt: function(index) {
return this._results[index];
},
/**
* Get the comment of the result at the given index
*/
getCommentAt: function(index) {
return this._comments[index];
},
/**
* Get the style hint for the result at the given index
*/
getStyleAt: function(index) {
if (!this._comments[index])
return null; // not a category label, so no special styling
if (index == 0)
return "suggestfirst"; // category label on first line of results
return "suggesthint"; // category label on any other line of results
},
/**
* Get the image for the result at the given index
* The return value is expected to be an URI to the image to display
*/
getImageAt : function (index) {
return "";
},
/**
* Remove the value at the given index from the autocomplete results.
* If removeFromDb is set to true, the value should be removed from
* persistent storage as well.
*/
removeValueAt: function(index, removeFromDb) {
this._results.splice(index, 1);
this._comments.splice(index, 1);
},
QueryInterface: function(aIID) {
if (!aIID.equals(Ci.nsIAutoCompleteResult) && !aIID.equals(Ci.nsISupports))
throw Components.results.NS_ERROR_NO_INTERFACE;
return this;
}
};
// Implements nsIAutoCompleteSearch
function SimpleAutoCompleteSearch() {
}
SimpleAutoCompleteSearch.prototype = {
classID: CLASS_ID,
classDescription: CLASS_NAME,
contractID: CONTRACT_ID,
/*
* Search for a given string and notify a listener (either synchronously
* or asynchronously) of the result
*
* #param searchString - The string to search for
* #param searchParam - An extra parameter
* #param previousResult - A previous result to use for faster searchinig
* #param listener - A listener to notify when the search is complete
*/
startSearch: function(searchString, searchParam, result, listener) {
// This autocomplete source assumes the developer attached a JSON string
// to the the "autocompletesearchparam" attribute or "searchParam" property
// of the <textbox> element. The JSON is converted into an array and used
// as the source of match data. Any values that match the search string
// are moved into temporary arrays and passed to the AutoCompleteResult
if (searchParam.length > 0) {
var nativeJSON = Components.classes["#mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
var searchResults = nativeJSON.decode(searchParam);
var results = [];
var comments = [];
for (i=0; i<searchResults.length; i++) {
if (searchResults[i].value.indexOf(searchString) == 0) {
results.push(searchResults[i].value);
if (searchResults[i].comment)
comments.push(searchResults[i].comment);
else
comments.push(null);
}
}
var newResult = new SimpleAutoCompleteResult(searchString, Ci.nsIAutoCompleteResult.RESULT_SUCCESS, 0, "", results, comments);
listener.onSearchResult(this, newResult);
}
},
/*
* Stop an asynchronous search that is in progress
*/
stopSearch: function() {
},
QueryInterface: function(aIID) {
if (!aIID.equals(Ci.nsIAutoCompleteSearch) && !aIID.equals(Ci.nsISupports))
throw Components.results.NS_ERROR_NO_INTERFACE;
return this;
},
_QueryInterface: XPCOMUtils.generateQI([Components.interfaces.nsIAutoCompleteSearch])
};
// Factory
var SimpleAutoCompleteSearchFactory = {
singleton: null,
createInstance: function (aOuter, aIID) {
if (aOuter != null)
throw Components.results.NS_ERROR_NO_AGGREGATION;
if (this.singleton == null)
this.singleton = new SimpleAutoCompleteSearch();
return this.singleton.QueryInterface(aIID);
}
};
// Module
var SimpleAutoCompleteSearchModule = {
registerSelf: function(aCompMgr, aFileSpec, aLocation, aType) {
aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
aCompMgr.registerFactoryLocation(CLASS_ID, CLASS_NAME, CONTRACT_ID, aFileSpec, aLocation, aType);
},
unregisterSelf: function(aCompMgr, aLocation, aType) {
aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
aCompMgr.unregisterFactoryLocation(CLASS_ID, aLocation);
},
getClassObject: function(aCompMgr, aCID, aIID) {
if (!aIID.equals(Components.interfaces.nsIFactory))
throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
if (aCID.equals(CLASS_ID))
return SimpleAutoCompleteSearchFactory;
throw Components.results.NS_ERROR_NO_INTERFACE;
},
canUnload: function(aCompMgr) { return true; }
};
// Module initialization
function NSGetModule(aCompMgr, aFileSpec) { return SimpleAutoCompleteSearchModule; }
if (XPCOMUtils.generateNSGetFactory){
var NSGetFactory = XPCOMUtils.generateNSGetFactory([SimpleAutoCompleteSearch]);
}
I added this code to chrome.manifest:
component {6224daa1-71a2-4d1a-ad90-01ca1c08e323} components/simpleautocomplete.js
contract #mozilla.org/autocomplete/search;1?name=simple-autocomplete {6224daa1-71a2-4d1a-ad90-01ca1c08e323}
In the xul file I added:
<textbox type="autocomplete" autocompletesearch="simple-autocomplete"
autocompletesearchparam='[{"value":"param1"},{"value":"param2"}]' />
important note: when setting the autocompletesearchparam attribute the assignment has to be insind two single quotes and not double quotes. only the values inside has to be in double quotes as in the xul above.
and that's it.
When I tested my extension and typed 'p' in the auto-complete textbox the words "param1" and "param2" poped-up.
I could also set the params in the js file as follows:
var searchTextField = document.getElementById("searchTextField");
var param1 = "Param1", param2 = "Param2";
paramsToSet = "[";
paramsToSet += "{\"value\" : \"" + param1 + "\"},";
paramsToSet += "{\"value\" : \"" + param2 + "\"},";
paramsToSet = paramsToSet.substring(0, paramsToSet.length-1); // to remove the last ","
paramsToSet += "]";
paramsToSet = paramsToSet.toLowerCase(); // important!
searchTextField.setAttribute("autocompletesearchparam", paramsToSet);
note1: notice that the auto-complete textbox only accepts text in small letters.
note2: if setting the params to the textbox is dynamically and taking some time, for example - when the client is typing in the textbox and the typed text is sent to a server so the response containing the params will be attached to the textbox, and this process is taking some time (about a second or half), then the popup auto-complete won't popup because it's autocompletesearchparam was empty when the client started to type. In this case it is possible to force the auto-complete to popup when you have the params to set in the textbox like this:
searchTextField.open = true;