I stored some jpeg files (exactly 350, same files same size. Total: 336.14 MB) as Blob in IndexedDB. It took around 1 second to complete the transaction. Then I read all the data from IndexedDB to an array and again sored to IndexedDB. But this time it takes around 15 Seconds. I observed this as a consistent behavior. Anything wrong here? I used performance.now() to get the time difference
Files: 350,
Size of each: 937 KB,
Browser: Chrome and Chromium Edge
//Open
var dbOpen = indexedDB.open(INDEXED_DB_NAME, INDEXED_DB_VERSION);
dbOpen.onupgradeneeded = function (e) {
console.log("onupgradeneeded");
var store = e.currentTarget.result.createObjectStore(
IMAGE_DATA_STORE, { autoIncrement: true });
};
dbOpen.onsuccess = function (e) {
image_data_db = dbOpen.result;
console.log("indexed DB opened");
};
//Initial Write
var inputFiles = document.getElementById('inputFiles');
for (var i = 0; i < inputFiles.files.length; i++) {
let file = inputFiles.files[i];
var b = new Blob([file], { type: file.type });
fileblobs.push(b);
}
StoreIdb(fileblobs); // < First write
//StoreIdb()
t0 = performace.now();
var trx = image_data_db.transaction(IMAGE_DATA_STORE, 'readwrite');
var imagestore = trx.objectStore(IMAGE_DATA_STORE);
for (i = 0; i < fileblobs.length; i++) {
request = imagestore.add(fileblobs[i]);
request.onsuccess = function (e) {
console.log('added');
};
request.onerror = function (e) {
console.error("Request Error", this.error);
};
}
trx.onabort = function (e) {
console.error("Exception:", this.error, this.error.name);
};
trx.oncomplete = function (e) {
console.log('completed');
t1 = performance.now();
timetaken = t1 - t0;
}
//Read
var objectStore = image_data_db.transaction(IMAGE_DATA_STORE).objectStore(IMAGE_DATA_STORE);
objectStore.openCursor().onsuccess = function (e) {
var cursor = e.target.result;
if (cursor) {
blobArray.push(cursor.value.blob);
cursor.continue();
}
else
{
// completed
}
}
// blobArray will be used for second time << Second Write
I figured it out. First time it was storing file instance blob.
I ve changed file instance blob to Array buffer just to want to ensure data type similar in both cases. Now it is taking same time.
for (var i = 0; i < inputFiles.files.length; i++) {
let file = inputFiles.files[i];
file.arrayBuffer().then((arrayBuffer) => {
let blob = new Blob([new Uint8Array(arrayBuffer)], {type: file.type });
blobs.push(blob);
if ( blobs.length == inputFiles.files.length){
callback(blobs);
}
});
}
I'm trying to upload multiple attachments.
First I'm getting attachments from user interface, then I'm converting them into JSON , then I need to make a server call.
In this I'm using FileReader.
//showing ajax loader
component.set("v.showLoadingSpinner", true);
//getting attached files
var files = component.find("fileId").get("v.files");
var details = {}; //JS Object need to send server
details.files = [];
for (var i = 0; i < files.length; i++)
{
(function(file) {
var name = file.name;
var reader = new FileReader();
reader.fName = files[i]['name'];
reader.fType = files[i]['type'];
reader.i = i;
reader.onload = function(e) {
var fileContents = reader.result;
var base64 = 'base64,';
var dataStart = fileContents.indexOf(base64) + base64.length;
fileContents = fileContents.substring(dataStart);
var startPosition = 0;
var endPosition = Math.min(fileContents.length, startPosition + 750000);
var getchunk = fileContents.substring(startPosition, endPosition);
var fDetails = {};
fDetails.fileName = reader.fName;
fDetails.base64Data = encodeURIComponent(getchunk);
fDetails.contentType = reader.fType;
details.files.push(fDetails);
}
reader.readAsDataURL(file);
})(files[i]);
// I want to make a server call here with data in "details" object.
console.log(details);
But I'm not getting data in above console log.
Please help me to achieve this.
You can use promises :
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise
https://davidwalsh.name/promises
https://developers.google.com/web/fundamentals/primers/promises
Also jQuery provide $.when() function :
https://api.jquery.com/jquery.when/
And with promisejs you can do something like this :
function readJSON(filename){
return new Promise(function (fulfill, reject){
readFile(filename, 'utf8').done(function (res){
try {
fulfill(JSON.parse(res));
} catch (ex) {
reject(ex);
}
}, reject);
});
}
I have created WebSocket.js but unfortunately I am unable to test it due to unavailability of data. The requirement is the data comes from various sources so I have multiple sockets for that. Note: I want to particularly test my socket.onMessage behaves for different sockets. Please find the code snippet below:
var webSocket;
var txQueue = [];
var defaultReconnectTimeout = 1000; //will be multiplied by 2 and saved into reconnectTimeout on each try
var reconnectTimeout = defaultReconnectTimeout;
var registerWebSocketHandlers = function(webSocket) {
webSocket.onclose = function(){
setTimeout(service.reopen, reconnectTimeout *= 2);
};
webSocket.onopen = function(e) {
reconnectTimeout = defaultReconnectTimeout; //reset this
deferredSend();
};
webSocket.onerror = function(e) {
throw new Error("[WebSocket] An error occured " + e);
};
}
var uniqid = function() {
return (new Date().getTime()).toString(16);
}
var deferredSend = function() {
if(!service.isOpen()) {
$timeout(deferredSend, 100);
return;
}
while(txQueue.length && service.isOpen()) {
var payload = txQueue.shift();
webSocket.send(typeof payload === 'string' ? payload : JSON.stringify(payload));
}
};
var createNewWebSocketInstance = function(apiUrl){
var websocket = new $window.WebSocket(apiUrl);
websocket.id = uniqid();
return websocket;
}
// TODO: this is a bit hacky since we directly bind it to the raw window event
$window.onbeforeunload = function(e) {
service.close();
};
var service = {};
service.setMessageEventHandler = function(name,cb) {
instances[name].onmessage = function(msg) {
if(msg.data.indexOf('Status: connected') === 0)
{
return;
}
var jsonObj = JSON.parse(msg.data);
cb(jsonObj);
};
};
service.isOpen = function() {
return webSocket.readyState === 1;
};
service.send = function(msg) {
txQueue.push(msg);
deferredSend();
};
service.close = function() {
return webSocket.close();
};
service.reopen = function() {
// get old message handler
var msgHandler = webSocket.onmessage;
// try closing the previous WebSocket
service.close();
// open new WebSocket
openConnection();
// re-attach old handler to new WebSocket
webSocket.onmessage = msgHandler;
};
service.getId = function() {
return webSocket.id;
}
// Returns an already existing instance of the socket, if unavailable then creates a new one.
service.getInstance = function(name, config) {
if(!(name in instances)) {
instances[name] = createNewWebSocketInstance(config);
}
registerWebSocketHandlers(instances[name]);
return instances[name];
};
return service;
You can test WebSocket using websocket.html at websocket.org Echo Test Creating your own test
Using a text editor, copy the following code and save it as
websocket.html somewhere on your hard drive. Then simply open it in a browser. The page will automatically connect, send a message,
display the response, and close the connection.
See Linux - WebSocket test error.
I have the following code snippet which I need to extend to define multiple WebSockets and I am clueless as to how do I go about it:
var registerWebSocketHandlers = function(webSocket) {
webSocket.onclose = function(){
setTimeout(service.reopen, reconnectTimeout *= 2);
};
webSocket.onopen = function(e) {
icc.publish('webSocket.reconnect');
reconnectTimeout = defaultReconnectTimeout; //reset this
deferredSend();
};
webSocket.onerror = function(e) {
throw new Error("[WebSocket] An error occured " + e);
};
}
var openConnection = function() {
connectionWasOpenBefore = true;
webSocket = new $window.WebSocket(xyz);
webSocket.id = uniqid();
registerWebSocketHandlers(webSocket);
};
var uniqid = function() {
return (new Date().getTime()).toString(16);
}
service.setMessageEventHandler = function(cb) {
webSocket.onmessage = function(msg) {
if(msg.data.indexOf('Status: connected') === 0)
{
return;
}
var jsonObj = JSON.parse(msg.data);
cb(jsonObj);
};
};
How do I twist the code to suit the needs of multiple WebSockets and attaching the appropriate callback to it?
Use the multiton pattern.
var socketFactory = module.factory('SocketFactory', function($rootScope){
var factory = {};
var instances = {};
factory.getInstance = function(name, config){
if(!(name in instances)){
instances[name] = createNewWebSocketInstance(name, config);
}
return instances[name];
};
var createNewWebSocketInstance = function(name, config){
var webSocket = new $window.WebSocket(config.address);
webSocket.id = uniqid();
registerWebSocketHandlers(webSocket, name, config.handlers); //etc.
return webSocket;
};
var registerWebSocketHandlers = function(webSocket, name, handlers){
webSocket.onmessage = function(event){
$rootScope.$emit('SocketMessageReceived_' + name, event.data);
};
//etc...
};
return factory;
});
This will separate your different websockets by name. Use getInstance('whatever') to get a websocket labelled as 'whatever'.
var firstConfig = {url: '*****', username: '****', password: '****', etc: '****'};
// You only need to pass in the config the first time.
var firstWebSocket = SocketFactory.getInstance('firstSocket', firstConfig);
var secondConfig = {url: '####', username: '####', password: '####', etc: '####'};
var secondWebSocket = SocketFactory.getInstance('secondSocket', secondConfig);
Next, from any other area you can access the configured websockets by their instance names.
var firstWebSocket = SocketFactory.getInstance('firstSocket');
// It would probably be a good idea to add this listener in the SocketFactory instead and broadcast an event when there's a message so multiple listeners can respond to it.
firstWebSocket.onmessage = function(){...};
var secondWebSocket = SocketFactory.getInstance('secondSocket');
secondWebSocket.onmessage = function(){...};
So, I'm developing a javascript + api forum software. My point in this is so that you can read a forum while offline -- this involves HTML5's offline-storage. In particular, I would like to use IndexedDB as it seems to be the most promising for the future. I've gotten a good service/factory for fetching / temporarily storing the data, but IDDB is broken majorly. Does anybody have advice on how to go about this?
edit Also, for anybody who would like a hosted version, here it is on cloud9.
var angular = angular || {};
(function(w){
w.localStorage = w.localStorage||{};
w.indexedDB = w.indexedDB || w.mozIndexedDB || w.webkitIndexedDB || w.msIndexedDB || null;
w.IDBTransaction = w.IDBTransaction || w.webkitIDBTransaction || w.msIDBTransaction || null;
w.IDBKeyRange = w.IDBKeyRange || w.webkitIDBKeyRange || w.msIDBKeyRange || null;
})(this);
angular.module("JSForumServices",[],function($provide){
$provide.factory('ForumStorage',(function(){
var service = {
post:null,
thread:null,
board:null,
cache:{},
pending:{}
};
var fetch = (function(baseFunction,path,callback){
if(path in service.pending)
return service.pending[path];
var r=baseFunction();
service.pending[path] = r;
var ajaxRequest = new XMLHttpRequest();
var cancelled = false;
var dateRegex =
/^(?:(Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s+)?(0[1-9]|[1-2]?[0-9]|3[01])\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(19[0-9]{2}|[2-9][0-9]{3})\s+(2[0-3]|[0-1][0-9]):([0-5][0-9])(?::(60|[0-5][0-9]))?\s+([-\+][0-9]{2}[0-5][0-9]|(?:UT|GMT|(?:E|C|M|P)(?:ST|DT)|[A-IK-Z]))(\s+|\(([^\(\)]+|\\\(|\\\))*\))*$/;
ajaxRequest.onreadystatechange = (function(){
var readyState = ajaxRequest.readyState;
if(readyState==4&&(!cancelled)){
// Store the copy locally!
// Also, initiate the callback.
// This way if the storage fails,
// The application continues to work
// As expected.
var data = JSON.parse(ajaxRequest.responseText);
for(var k in data)
r[k] = data[k];
service.cache[path]={obj:r,modified:new Date()};
delete service.pending[path];
callback(r);
}
else if((path in service.cache)&&readyState>=2){
var oldDate = service.cache[path].modified;
console.log("Cache'd copy for",path,"exists.",oldDate.toString());
var isMoreRecent = false;//Is the server-copy more recent?
var serverModifiedString = ajaxRequest.getResponseHeader("Last-Modified");
console.log(serverModifiedString);
var match = dateRegex.exec(serverModifiedString);
var serverModified = new Date();
serverModified.setDate(parseInt(match[2],10));
serverModified.setMonth({
"jan":0,
"feb":1,
"mar":2,
"apr":3,
"may":4,
"jun":5,
"jul":6,
"aug":7,
"sep":8,
"oct":9,
"nov":10,
"dec":11
}[match[3].toLowerCase()]);
serverModified.setYear(parseInt(match[4],10));
serverModified.setHours(parseInt(match[5],10));
serverModified.setMinutes(parseInt(match[6],10));
serverModified.setMilliseconds(parseInt(match[7],10));
isMoreRecent = serverModified > oldDate;
if(!isMoreRecent&&(path in service.pending)){//sometimes this code may be slower than network speeds? Just to be safe, I guess.
cancelled=true;
var oldObject = service.cache[path].obj;
for(var key in oldObject)
r[key]=oldObject[key];
console.log("using a cache'd value.",r);
callback(r);
delete service.pending[path];
ajaxRequest.abort();//No need to waste more bandwidth!
}
}
});
ajaxRequest.open("GET",path,true);
ajaxRequest.send();
return r;
});
var ObjectFetch = (function(base,constr){
var ret = (function(id,cb){
cb = cb||(function(){});
return fetch(constr,base+id+".json",cb);
});
return ret;
});
service.post = ObjectFetch("./post/",(function(){
return {
id:"???",
author:"???",
content:"",
date:""
};
}));
service.thread = ObjectFetch("./thread/",(function(){
return {
id:"???",
title:"???",
posts:[],
tags:""
};
}));
service.board = ObjectFetch("./board/",(function(){
return {
id:"???",
title:"???",
threads:[],
tags:""
};
}));
service.forum = ObjectFetch("./",(function(){
return {
name:"Javascript Forum Software.",
boards:[]
};
}));
if(window.indexedDB!==null){
var postFetch = service.post;
var threadFetch = service.thread;
var boardFetch = service.board;
(function(dbengine){
var boardsLoaded = false;
var threadsLoaded = false;
var postsLoaded = false;
var req = dbengine.open("forum",1);
req.onupgradeneeded = (function(e){
var db = e.target.result;
db.createObjectStore("post",{
keyPath:"id"
});
db.createObjectStore("thread",{
keyPath:"id"
});
db.createObjectStore("board",{
keyPath:"id"
});
});
req.onsuccess = (function(e){
service.database = e.target.result;
var loadData = service.database.transaction([
'board',
'thread',
'post'],'readwrite');
loadData.onsuccess = (function(ee){
var transaction = ee.target.result;
transaction.objectStore("board").openCursor().onsuccess=(function(e){
var cursor = e.target.result;
if(cursor===null){
boardsLoaded = true;
return;
}
var id = cursor.key;
var obj = cursor.value;
var lastModified = new Date();
if("lastModified" in obj){
lastModified = obj.lastModified;
}
service.cache["./board/"+id.toString().toLowerCase()+".json"]={
obj:obj,
modified:lastModified
};
});
transaction.objectStore("thread").openCursor().onsuccess=(function(e){
var cursor = e.target.result;
if(cursor===null){
threadsLoaded = true;
return;
}
var id = cursor.key;
var obj = cursor.value;
var lastModified = new Date();
if("lastModified" in obj){
lastModified = obj.lastModified;
}
service.cache["./thread/"+id.toString().toLowerCase()+".json"]={
obj:obj,
modified:lastModified
};
});
transaction.objectStore("post").openCursor().onsuccess=(function(e){
var cursor = e.target.result;
if(cursor===null){
postsLoaded = true;
return;
}
var id = cursor.key;
var obj = cursor.value;
var lastModified = new Date();
if("lastModified" in obj){
lastModified = obj.lastModified;
}
service.cache["./post/"+id.toString().toLowerCase()+".json"]={
obj:obj,
modified:lastModified
};
});
service.post = (function(id,cb){
console.log("DDDDDAFF");
var trans = service.database.transaction(["post"],"readwrite");
trans.onsuccess = (function(e){
var req = e.target.result.objectStore("post").get(id);
req.onsuccess = (function(ee){
cb(req.result);
});
req.onerror = (function(ee){
console.log("HAAAA?!");
postFetch(id,(function(post){
e.target.result.objcetStore.save(post);
cb(post);
}));
});
});
trans.onerror = (function(e){
console.log("Error with IDDB:",e);
threadFetch(id,cb);
});
});
service.thread = (function(id,cb){
var trans = service.database.transaction(["thread"],"readwrite");
trans.onsuccess = (function(e){
var req = e.target.result.objectStore("thread").get(id);
req.onsuccess = (function(ee){
cb(req.result);
});
req.onerror = (function(ee){
threadFetch(id,(function(post){
e.target.result.objcetStore.save(post);
cb(post);
}));
});
});
trans.onerror = (function(e){
console.log("Error with IDDB:",e);
postFetch(id,cb);
});
});
service.board = (function(id,cb){
var trans = service.database.transaction(["board"],"readwrite");
trans.onsuccess = (function(e){
var req = e.target.result.objectStore("board").get(id);
req.onsuccess = (function(ee){
cb(req.result);
});
req.onerror = (function(ee){
boardFetch(id,(function(post){
e.target.result.objcetStore.save(post);
cb(post);
}));
});
});
trans.onerror = (function(e){
console.log("Error with IDDB:",e);
boardFetch(id,cb);
});
});
});
});
})(window.indexedDB);
}
return service;
}));
});
angular.module('JSForum',["JSForumServices"]).config(
['$routeProvider',
function($routeProvider){
$routeProvider.when('/',{
templateUrl:"forum.html",
controller:ForumController
});
$routeProvider.when('/board/:id',{
templateUrl:"board.html",
controller:BoardController
});
$routeProvider.when('/thread/:id',{
templateUrl:"thread.html",
controller:ThreadController
});
$routeProvider.otherwise({redirectTo:"/"});
}]);
function ThreadController($scope,$routeParams,ForumStorage){
$scope.id = $routeParams.id.toString().toLowerCase();
$scope.thread = null;
ForumStorage.thread($scope.id,(function(thread){
$scope.thread = thread;
$scope.$apply();
var callback = (function(p){
return (function(post){
$scope.thread.posts[p] = post;
$scope.$apply();
});
});
for(var i=0;i<thread.posts.length;i++)
if(typeof(thread.posts[i].id) == 'undefined')
ForumStorage.post(thread.posts[i],callback(i));
$scope.$apply();
}));
}
function BoardController($scope,$routeParams,ForumStorage){
$scope.id = $routeParams.id.toString().toLowerCase();
$scope.board = null;
ForumStorage.board($scope.id,(function(board){
var callback = (function(p){
return (function(thread){
$scope.board.threads[p] = thread;
$scope.$apply();
});
});
$scope.board = board;
console.log("Using board:",$scope.board);
for(var i=0;i<board.threads.length;i++)
if(typeof(board.threads[i].id)=='undefined')
ForumStorage.thread(board.threads[i],callback(i));
$scope.$apply();
}));
}
function ForumController($scope,ForumStorage){
$scope.name = localStorage.forumName||"Forum";
$scope.boards = [];
ForumStorage.forum("forum",(function(forum){
document.title = $scope.name = localStorage.forumName = forum.name;
$scope.boards = forum.boards;
var callback=(function(p){
return (function(o){
$scope.boards[p] = o;
$scope.$apply();
});
});
for(var i=0;i<$scope.boards.length;i++){
if(typeof($scope.boards[i].id) == 'undefined')
ForumStorage.board(forum.boards[i],callback(i));
}
$scope.$apply();
}));
}
If your backend is google cloud storage, you can use my open source database library http://dev.yathit.com/api-reference/ydn-db/storage.html It cached in IndexedDB and persist to blob store. forum post URI path is taken as primary key of the record. LastModified header value is persisted into IndexedDB and used for conditional HTTP request. since blob store can only be queried by ascending order of key (URI path), forum post URI path are generated so that the last post is the smallest. In this way, we can query new posts by giving smallest known key as marker.
An example can be found in https://bitbucket.org/ytkyaw/ydn-auth/src/master/examples/note-app/note-app.js (not forum, but a simple note app). you need a backend server that generate signed url for posting new record.