I am wondering why trying to run the following test suite fails when I try to delete the table I have stored entities in. The error I get is the following
1) Azure Storage cloud storage operations "after all" hook:
Error: The specified resource does not exist. RequestId:3745d709-fa5e-4a2b-b517-89edad3efdd2
Time:2013-12-03T22:26:39.5532356Z
If I comment out the actual insertion of data it fails every other time, and if I try to do the insertion of data it fails every time with an additional "The table specified does not exist.".
For the first case this seems to indicate that there is some kind of delay in the table creation, so in every other test it is successful, and for the second case it seems to indicate that even though my callbacks are being called after table creation, the table(s) still aren't ready for data insertion.
The test suite and associated code looks like this:
describe('cloud storage operations', function () {
var storage;
before(function (done) {
this.timeout(5000);
storage = AzureStorage.usingTable('TEST', done);
});
after(function (done) {
storage.deleteTable(done);
});
it('should store without trouble', function (done) {
storage.save(factory.createChangeSet()).then(done, done);
});
});
... // snipped from azure.js
var AzureStorage = function (storageClient, tableName, callback) {
assert(storageClient && tableName && partitionKey, "Missing parameters");
this.storageClient = storageClient;
this.tableName = tableName;
var defaultCallback = function (err) { if (err) { throw error; } };
this.storageClient.createTableIfNotExists(this.tableName, function () {
callback();
} || defaultCallback);
};
AzureStorage.usingTable = function (tableName, callback) {
return new AzureStorage(
azure.createTableService(accountName, accountKey)
, tableName
, callback
);
};
AzureStorage.prototype.deleteTable = function (callback) {
this.storageClient.deleteTable(this.tableName, callback);
};
I've hit this using the c# library as well but I'm pretty sure the error message indicated the table could not be created because an operation was still in process for a table of the same name. Thinking of the backend supporting storage, it makes sense that it would not be instant. The table needs to be removed from the 3 local replicas as well as the replicas in the paired data center.
With that kind of async operation, it is going to be challenging to build up an tear them down fast enough for tests.
A workaround might be to increment a value appended to the "TEST" table name that would be unique to that test run.
Related
I'm new to the idea of asynchronous code, and am still trying to wrap my brain around how everything works.
I'm building a Node Express application which will interface with a database. When running in a development environment I want it to interface with a Sqlite database. (The production database will not use Sqlite. This only applies to creating a small development environment.)
My problem is I'm having trouble controlling the execution order and timing of queries to the database.
I would like to build my SqliteDatabase.js file such that it can only execute queries sequentially, despite the fact that functions in this file will be called by other parts of the program that are running asynchronously.
How can I acheive this?
For reference, here is how I currently have my SqliteDatabase.js file set up:
var debug = require('debug')('app:DATABASE');
var sqlite = require('sqlite3').verbose();
open = function() {
var db = new sqlite.Database('./test-database.db', sqlite.OPEN_READWRITE | sqlite.OPEN_CREATE, function(err) {
if (err) {
debug("We've encountered an error opening the sqlite database.");
debug(err);
} else {
debug("Sqlite database opened successfully.");
}
});
return db;
}
executeSQLUpdate = function(sql, next) {
var db = open();
db.serialize(function() {
console.log("executing " + sql);
db.run(sql);
db.close();
next();
});
}
exports.executeSQLUpdate = executeSQLUpdate;
Is there some way to build a queue, and make it so when the "executeSQLUpdate" function is called, the request is added to a queue, and is not started until all previous requests have been completed?
To give an example, take a look at this code which utilises my SqliteDatabase.js file:
ar database = require('../../bin/data_access/SqliteDatabase.js');
var createTestTableStmt = "CREATE TABLE IF NOT EXISTS Test(\n" +
"Name TEXT PRIMARY KEY NOT NULL UNIQUE,\n" +
"Age INT NOT NULL,\n" +
"Gender TEXT NOT NULL\n" +
");";
var clearTestTableStmt = "DELETE FROM Test;";
var testInsertStmt = "INSERT INTO Test (Name, Age, Gender)\n" +
"VALUES (\"Connor\", 23, \"Male\");";
createTable = function() {
database.executeSQLUpdate(createTestTableStmt, clearTable);
}
clearTable = function() {
database.executeSQLUpdate(clearTestTableStmt, insertRow);
}
insertRow = function() {
database.executeSQLUpdate(testInsertStmt, function() {
console.log("Done!");
});
}
createTable();
9 times out of 10 the above code works fine, but every once in a while, the "insert row" function is called before the "clearTable" function is called, which throws an error because of a violated database constraint.
How can I change my implementation of the SqliteDatabase.js file to avoid this issue?
You can use async to do this using await. This code will wait for each asynchronous database call to complete before executing the next line.
async function createTable() {
await database.executeSQLUpdate(createTestTableStmt);
await database.executeSQLUpdate(clearTestTableStmt);
await database.executeSQLUpdate(testInsertStmt);
console.log("Done!");
}
Your console.log statement will only execute once all three have completed.
I should also mention that you need a try...catch block around the three database calls to trap any errors and provide an alternate exit point if something should go wrong.
I realized why the callback function next() was sometimes being called before db.run(sql)
It turns out that db.run() is itself an asychronous function. I updated my code, and added a callback to the db.run() line to make sure we don't skip ahead until it's done.
Here's what it looks like now:
executeSQLUpdate = function(sql, next) {
var db = open();
db.run(sql, function(err) {
db.close(function() {
if (next) next(err);
});
});
}
Nesting each asynchronous function in the previous function's callback, makes each function execute in order.
Thanks to everyone who gave me hints that helped me figure out what the problem was.
I'd like to write a feature like this:
Scenario: new Singleton create
When a new, unmatchable identity is received
Then a new tin record should be created
And a new bronze record should be created
And a new gold record should be created
which would tie to steps like this:
defineSupportCode(function ({ Before, Given, Then, When }) {
var expect = require('chai').expect;
var chanceGenerator = require('./helpers/chanceGenerator')
var request = require('./helpers/requestGenerator')
let identMap;
// reset identMap before each scenario
Before(function () {
identMap = [];
});
// should generate a valid identity
// persist it in a local variable so it can be tested in later steps
// and persist to the db via public endpoint
When('a new, unmatchable identity is received', function (callback) {
identMap.push(chanceGenerator.identity());
request.pubPostIdentity(identMap[identMap.length-1], callback);
});
// use the local variable to retrieve Tin that was persisted
// validate the tin persisted all the props that it should have
Then('a new tin record should be created', function (callback) {
request.pubGetIdentity(identMap[identMap.length-1], callback);
// var self = this;
// request.pubGetIdentity(identMap[identMap.length-1], callback, () => {
// console.log('never gets here...');
// self.callback();
// callback();
// });
// request.pubGetIdentity(identMap[identMap.length-1], (callback) => {
// console.log('never gets here...');
// self.callback();
// callback();
// });
});
The issue that I'm having is that I can't do anything in the Then callback. That is where I'd like to be able to verify the response has the right data.
Here are relevant excerpts from the helper files:
var pubPostIdentity = function (ident, callback) {
console.log('pubIdentity');
var options = {
method: 'POST',
url: 'http://cucumber.utu.ai:4020/identity/' + ident.platform + '/' + ident.platformId,
headers: {
'X-Consumer-Custom-Id': ident.botId + '_' + ident.botId
},
body: JSON.stringify(ident)
};
console.log('ident: ', ident);
request(options, (err, response, body) => {
if (err) {
console.log('pubPostIdentity: ', err);
callback(err);
}
console.log('pubPostIdentity: ', response.statusCode);
callback();
});
}
// accept an identity and retrieve from staging via identity public endpoint
var pubGetIdentity = function (ident, callback) {
console.log('pubGetIdentity');
var options = {
method: 'GET',
url: 'http://cucumber.utu.ai:4020/identity/' + ident.platform + '/' + ident.platformId,
headers: {
'X-Consumer-Custom-Id': ident.botId + '_' + ident.botId
}
};
request(options, (err, response) => {
if (err) {
console.log('pubGetIdentity: ', err);
callback(err);
}
console.log('pubGetIdentity: ', response.body);
callback();
});
}
Something that we are considering as an option is to re-write the feature to fit a different step definition structure. If we re-wrote the feature like this:
Scenario: new Singleton create
When a new, unmatchable 'TIN_RECORD' is received
Then the Identity Record should be created successfully
When the Identity Record is retreived for 'tin'
Then a new 'tin' should be created
When the Identity Record is retreived for 'bronze'
Then a new 'bronze' should be created
When the Identity Record is retreived for 'gold'
Then a new 'gold' should be created
I believe it bypasses the instep callback issue we are wrestling with, but I really hate the breakdown of the feature. It makes the feature less readable and comprehensible to the business.
So... my question, the summary feature presented first, is it written wrong? Am I trying to get step definitions to do something that they shouldn't? Or is my lack of Js skills shining bright, and this should be very doable, I'm just screwing up the callbacks?
Firstly, I'd say your rewritten feature is wrong. You should never go back in the progression Given, When, Then. You are going back from the Then to the When, which is wrong.
Given is used for setting up preconditions. When is used for the actual test. Then is used for the assertions. Each scenario should be a single test, so should have very few When clauses. If you want, you can use Scenario Outlines to mix several very similar tests together.
In this case, is recommend to take it back to first principles and see if that works. Then build up slowly to get out working.
I suspect in this case that the problem is in some exception being thrown that isn't handled. You could try rewriting it to use promises instead, which will then be rejected on error. That gives better error reporting.
Working on the IndexedDB API, I'm creating many objectStores that belong to the same database, in one transaction, when the user loads a webpage.
I order to do so, I created an object which contains many objectStores to be created, each one has it's name, data and index.
Then a function runs the object and effectively creates Database, objectStores and indexes for each one.
However of all OS's created, just the last member of the object gets populated. Say of 5 objects to be created and populated, 5 are created but only the last one is populated.
Clearly is a problem of overwriting or some issue related to the JS stack or asynchronicity.
I appreciate any help to make the code populate all OS not the last one.
My browser is Chrome 56, I fetch data from an API whose response is OK, and I'm coding on vanillajs. I appreciate your help in vanillajs, there is no way to use any library or framework different from what the modern Web Platform offers.
Here is the code:
On the HTML side, this is an example of the object:
var datastores = [{osName:'items', osEndpoint: '/api/data/os/1/1', osIndex:'value'}, {osName:'categories', osEndpoint: '/api/data/os/2/1', osIndex: 'idc'}];
On javascript:
var request = indexedDB.open(DB_NAME, DB_VERSION); // open database.
request.onerror = function (e) { // error callback
console.error("error: " + e.target.errorCode);
};
request.onupgradeneeded = function (e) { // the onupgradeneeded event which creates all schema, dataabase, objectstores and populates OS.
var db = this.result;
for (var i in datastores) { // loop the objectStore object.
var objectStore = db.createObjectStore(datastores[i].osName, {keyPath: "id"});
TB_NAME = datastores[i].osName; // instantiate each objectStore name.
objectStore.createIndex(datastores[i].osIndex, datastores[i].osIndex, { unique: false }); // create each index.
objectStore.transaction.oncomplete = function(e) { // oncomplete event, after creating OS...
fetchGet(datastores[i].osEndpoint, popTable); // runs a function to fetch from a designated endpoint and calls a function.
};
}
}
Now the functions: to fetch data and to populate data:
function fetchGet(url, function) { // fetch from API.
fetch(url, {
method: 'GET'
}).then(function(response) {
return response.json();
}).then(function(json) {
popTable (json);
}).catch(function(err) {
console.log('error!', err);
});
}
function popTable(json) {
var m = 0;
var tx = db.transaction(TB_NAME, "readwrite");
tx.oncomplete = function(e) {
console.log("Completed Transaction " + TB_NAME);
};
tx.onerror = function(e) {
console.error("error: " + e.target.errorCode);
};
var txObjectStore = tx.objectStore(TB_NAME);
for (m in json) {
var request = txObjectStore.add(json[m]);
request.onsuccess = function (e) {
console.log('adding... ' );
};
}
}
The for (var i in datastores) loop runs synchronously, updating the global TB_NAME variable every time. When the loop finishes, TB_NAME will be holding the name of the last object store.
By the time the asynchronous popTable calls run, TB_NAME will forever be holding the name of the last store, so that's the only one that will update. Try adding logging to popTable to see this.
You'll need to pass the current value of the store name along somehow (e.g. as an argument to fetchGet). Also note that although you pass popTable as a parameter when calling fetchGet you're not actually accepting it as an argument.
...
Specific changes:
Change how you call fetchGet to include the store name:
fetchGet(datastores[i].osEndpoint, popTable, datastores[i].osName);
Change the fetchGet function to accept the args:
function fetchGet(url, func, name) {
And then instead of calling popTable directly, do:
func(json, name);
And then change the definition of popTable to be:
function popTable(json, name) {
... and use name in the transaction.
I was recently building a scraper module to get some information with nodejs until I encountered this "little" problem. The modules that I'm using are cheeriojs and request.
Actually the module works like a charm if I call only one method at a time. It contains three function and only two of them are exported, this is the code:
'use strict';
var request = require('request'),
cheerio = require('cheerio'),
counter = 0;
function find(term, cat, callback) {
// All the check for the parameters
scrape("http://.../search.php?search=" + encodeURIComponent(term), cat, callback);
}
function last(cat, callback) {
// All the check for the parameters
scrape("http://google.com/", cat, callback);
}
function scrape(url, cat, callback) {
request(url, function (error, response, body) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(body);
var result = [];
var items = $('.foo, .foo2').filter(function() {
// Condition to filter the resulted items
});
items.each(function(i, row) {
// Had to do another request inside here to scrape other information
request( $(".newpagelink").attr("href"), function(error, response, body) {
var name = $(".selector").text(),
surname = $(".selector2").text(),
link = cheerio.load(body)('.magnet').attr('href'); // This is the only thing that I'm scraping from the new page, the rest comes from the other "cheerio.load"
// Push an object in the array
result.push( { "name": name, "surname": surname, "link": link } );
// To check when the async requests are ended
counter++;
if(counter == items.length-1) {
callback(null, result);
}
});
});
}
});
}
exports.find = find;
exports.last = last;
The problem now, as I was saying, is that if I create a new node script "test.js" and I call only last OR find, it works perfectly! But if I call both the methods consecutively like this:
var mod = require("../index-tmp.js");
mod.find("bla", "blabla", function(err, data) {
if (err) throw err;
console.log(data.length + " find");
});
mod.last(function(err, data) {
console.log(data.length + " last");
});
The results are completely messed up, sometimes the script doesn't even print something, other times print the result of only "find" or "last", and other times returns a cheeriojs error (I won't add here to not mess you up, because probably it's my script's fault). I thought also to repeat the same function two times for both the methods but nothing, the same problems occur... I don't know what else to try, I hope you'll tell me the cause of this behavior!
Your counter variable is global, not specific to each scrape call. It wouldn't work if you called find twice at the same time either, or last.
Move the declaration and initialisation of var counter = 0; into the scrape function, or even better right next to the result and items declarations.
From scanning your code quickly, this is probably due to the variable counter being global. These are asynchronous functions, so they will both act on counter at the same thing. Move the declaration inside of the scrape function.
If you need more information about asynchronous programming, refer to Felix's great answer in this question.
I am using Sails v0.11 and am developing an standalone importer script in order to import data to mongoDB and - that is now the not-working part - build the associations between the models.
For this process I introduced temporary helper properties in the models in order to find the associated records and replace them by in real MongoDB _ids.
The script starts Sails in order to be able use its features (waterline, etc.):
var app = Sails();
app.load({
hooks: { grunt: false },
log: { level: 'warn' }
}, function sailsReady(err){
processUsers() finds all users and their _ids and iterates over them to invoke a second function addOrgsToOneUser()
var processUsers = function() {
// Iterate through all users in order to retrieve their _ids and
app.models['user'].native(function(err, collection) {
collection.find({}, projectionOrgInUser).toArray(function (err, users) {
Async.eachSeries(users, function (user, next){
// prepare userInOrgs
whereUserInOrg = { orgId: { $in: userInOrgs } };
//This is invoking
addOrgsToOneUser(user, whereUserInOrg);
next();
}, function afterwards (err) {
if (err) {
console.error('Import failed, error details:\n',err);
return process.exit(1);
}
console.log("done");
return process.exit(0); // This returns too early, not executing the addOrgsToOneUser
});
});
});
};
addOrgsToOneUser() finds all orgs belonging to THIS user and updates then the orgs array property of THIS user
var addOrgsToOneUser = function(user, whereUserInOrg) {
var projectionUserInOrg = "...";
// Find all orgs that this user is associated to and store it in inOrgs
app.models['org'].native(function(err, collection) {
collection.find(whereUserInOrg, projectionUserInOrg).toArray(function (err, orgs) {
// prepare inOrgs which is needed for updating
//update user to have an updated orgs array based on inOrgs.
app.models['user'].update({'id' : user._id.toString()}, {'orgs': inOrgs}).exec(function afterwards(err, updated){
console.log('Updated user ' + user._id.toString() + ' to be in their orgs');
});
});
});
}
Problem:
Process.exit(0) is called before the query/update of saddOrgsToOneUser() has completed. It behaves as expected if saddOrgsToOneUser() contains just a console.log for instance, but queries are triggered ansynchronously of course.
In case I comment out Process.exit(0), the script never stops, but the queries are executed as intented.
As the script will have further nested queries, I need a better approach to this as manually kill this script ...
How is nesting queries and iterating over their results done properly?
Thank you very much,
Manuel
addOrgsToOneUser is asynchronous. next() needs to be called after everything is done inside addOrgsToOneUser. The way I would do it is to pass in a callback (next) and call it when everything is done. So the call is
addOrgsToOneUser(user, whereUserInOrg, next);
and the addOrgsToOneUser will have an extra argument:
var addOrgsToOneUser = function(user, whereUserInOrg, callback) {
var projectionUserInOrg = "...";
// Find all orgs that this user is associated to and store it in inOrgs
app.models['org'].native(function(err, collection) {
collection.find(whereUserInOrg, projectionUserInOrg).toArray(function (err, orgs) {
// prepare inOrgs which is needed for updating
//update user to have an updated orgs array based on inOrgs.
app.models['user'].update({'id' : user._id.toString()}, {'orgs': inOrgs}).exec(function afterwards(err, updated){
console.log('Updated user ' + user._id.toString() + ' to be in their orgs');
callback(); // your original next() is called here
});
});
});
}