Well say I have a database, which is a specific "mock" for testing purposes, hence an memory based sqlite database -- where to main database uses postgres:
import knexBuilder = require("knex");
const theDb = knexBuilder({
client: 'sqlite3',
connection: {
filename: ":memory:"
},
useNullAsDefault: true,
pool: {
min: 1,
max: 1,
idleTimeoutMillis: 360000 * 1000
},
});
await theDb.schema.createTable("test", (table) => {
table.increments('id');
table.json("json_test").nullable();
});
When I call column info on this await theDb.table('test').columnInfo();
this is the response:
{
"id":{"type":"integer","maxLength":null,"nullable":false,"defaultValue":null},
"json_test":{"type":"json","maxLength":null,"nullable":true,"defaultValue":null}
}
However when I try to do an insert-retrieve circle like this:
await theDb.insert({
id: 1,
test_json: {
"abc": 10,
"def": "hello",
"ghi": 20,
},
}).into("test");
const retrieved = await theDb.select('id', 'test_json').from('test').where('id', 1);
The result is: [{id: 1, test_json: null}];. Which seems to indicate that the json is not stored? Printing the query with "toString()" shows it does put the json in the values:
insert into `test` (`id`, `test_json`) values (1, {"abc":10,"def":"hello","ghi":20})
I notice that SQLITE has by default no support for JSON, however there is a supported JSON extension for SQLite: https://www.sqlite.org/json1.html
So how would I tell knex "json is fine, just use the extension"? Or any other idea where I can use json without code breaking for tests? The rest of the code depends on JSON and it would be kind of silly to change it everywhere to use text/JSON serializing "if ran during testing".
Related
I want to be able to query elements in a redis cache based on 3 different indexes. Those indexes would be:
A MAC address stored as a String.
A number.
A latitude and longitude(to be able to query spatially).
I have seen that Redis has support for multi indexing using redis search and native geospatial api.
so using nodejs and node-redis I have written the following index:
client.ft.create(
'idx:cits',
{
mid: {
type: SchemaFieldTypes.TEXT
},
timestamp: {
type: SchemaFieldTypes.NUMERIC,
sortable: true
},
position: {
type: SchemaFieldTypes.GEO
}
},
{
ON: 'HASH',
PREFIX: 'CITS'
}
)
Now, i would like to insert records on the database that include those 3 parameters plus an additional String that stores some payload. I have tried using
await client.hSet('CITS:19123123:0:0:00:00:5e:00:53:af', {
timestamp: 19123123,
position: {latitude:0, longitude:0},
mid: '00:00:5e:00:53:af',
message: 'payload'
})
But I get the following error:
throw new TypeError('Invalid argument type');
^
TypeError: Invalid argument type
So, i can't add the latitude and longitude that way, I also tried
using the module ngeohash and computing an 11 character wide geohash like so:
await client.hSet('CITS:19123123:0:0:00:00:5e:00:53:af', {
timestamp: 19123123,
position: geohash.encode(0, 0, 11),
mid: '00:00:5e:00:53:af',
message: 'payload'
})
And it does not give any error but when using redis search querys It does not find points near it.
Is it even possible what I am trying to do? If so, how would you input the data to the redis database?
Here is a minimal reproducible example (Im using "ngeohash": "^0.6.3" and "redis": "^4.5.0"):
const { createClient, SchemaFieldTypes } = require('redis')
const geohash = require('ngeohash')
const client = createClient()
async function start(client) {
await client.connect()
try {
// We only want to sort by these 3 values
await client.ft.create(
'idx:cits',
{
mid: {
type: SchemaFieldTypes.TEXT
},
timestamp: {
type: SchemaFieldTypes.NUMERIC,
sortable: true
},
position: {
type: SchemaFieldTypes.GEO
}
},
{
ON: 'HASH',
PREFIX: 'CITS'
}
)
} catch (e) {
if (e.message === 'Index already exists') {
console.log('Skipping index creation as it already exists.')
} else {
console.error(e)
process.exit(1)
}
}
await client.hSet('CITS:19123123:0:0:00:00:5e:00:53:af', {
timestamp: 19123123,
position: geohash.encode(0, 0, 11),
mid: '00:00:5e:00:53:af',
message: 'payload'
})
await client.hSet('CITS:19123123:0.001:0.001:ff:ff:ff:ff:ff:ff', {
timestamp: 19123123,
position: geohash.encode(0.001, 0.001, 11),
mid: 'ff:ff:ff:ff:ff:ff',
message: 'payload'
})
const results = await client.ft.search(
'idx:cits',
'#position:[0 0 10000 km]'
)
console.log(results)
await client.quit()
}
start(client)
Additionally, I would like to ask if there is maybe another type of database that better suits my needs. I have chosen redis because it offers low latency, and that is the biggest constraint in my environment(I will probably do more writes than reads per second). I only want it to act as a inmediate cache, as persistent data will be stored in another database that does not need to be fast.
Thank you.
You get the Invalid argument type error because Redis does not support nested fields in hashes.
"GEO allows geographic range queries against the value in this attribute. The value of the attribute must be a string containing a longitude (first) and latitude separated by a comma" (https://redis.io/commands/ft.create/)
I'm trying to fetch contents inside a script tag from a url using node-fetch and then trying to json parse the data but i keep getting a return undefined.
I'm trying to get the content from the variable game from the html below and then stringify and then parse the json but it returns undefined.
Page html:
<!DOCTYPE html>
<head>
<meta charset="UTF-8" />
<title>Document Title</title>
</head>
<body>
<div id="welcome-div">
<p>Welcome to the my website</p>
</div>
<script>
var game = new Game({
stage: "prod",
servers: {
v32306117: {
id: "v32306117",
name: "name #1",
hostname: "hostname1",
port: 80,
},
v32306125: {
id: "v32306125",
name: "name #2",
hostname: "hostname2",
port: 80,
}
},
userGroup: 0
});
game.init(function() {
game.assetManager.load([{
"name": "\/asset\/image\/map\/grass.png",
"url": "\/asset\/image\/map\/grass.png"
}]);
game.debug.init();
game.run();
});
</script>
</body>
</html>
Fetch function:
const fetch = require("node-fetch");
async function serversFetch() {
try {
const servers = await fetch("https://get-servers.herokuapp.com/");
const data = await servers.text();
const servers_data = data.substring(
data.lastIndexOf("var game =") + 20,
data.lastIndexOf("game.init") - 10
);
return JSON.stringify(servers_data);
} catch (error) {
console.log(error);
}
}
(async () => {
const data = await serversFetch();
console.log('data', data);
const info = JSON.parse(data);
console.log('info', info.servers); // returns undefined
})()
if i console log info.servers it comes back undefined but if i console log just info it logs the output below.
info {
stage: "prod",
servers: {
v32306117: {
id: "v32306117",
name: "name #1",
hostname: "hostname1",
port: 80,
},
v32306125: {
id: "v32306125",
name: "name #2",
hostname: "hostname2",
port: 80,
}
},
userGroup: 0
}
The issue you are running into is because JSON.stringify only works on JavaScript objects and servers_data is a string. What this results in is info being a string later on and that's why console.log(info.servers) logs undefined. When you checked console.log(info), it only had the appearance of working correctly since it was logging the string value of the object. You could test this by performing console.log(typeof info) and you'll see it's of type string.
What you are looking for is to have servers_data be a valid JSON string instead of being a string of a JavaScript object (which is missing all those double-quotes around object property names which JSON requires). The first option might be to brute force it and replace those properties with properties wrapped in double-quotes, i.e. servers: for "servers:" (the colon is included to make it more unique, but it's still not bulletproof). That doesn't help you with properties like v32306117 which are likely unique and can't be replaced easily using the brute force replacement which is looking for known properties.
The next option would likely be to create a parser for the string which could parse it into an abstract syntax tree (AST) for JavaScript. You could then map that to the AST of JSON easily and then reform it as a string using a parser which understands the AST of JSON. Most parsers which use ASTs can parse a string to an AST and create a string from an AST. These parsers are often written using a recursive descent parser. Though this is a great exercise for programmers, you could likely find some libraries which implement AST parsers for JavaScript and JSON. Also, it's a bit of overkill with what you are trying to accomplish.
The final option which I think would be the easiest to implement and easiest to maintain would be to use JSON5, which is a superset of JSON. Using JSON5.parse, you could parse servers_data as is and not have to worry about JSON formatting. This is because JSON5 accepts properties without the double-quotes and is much more lenient about the formatting (double-quotes vs single-quotes, etc.).
EDIT: Re-structured question, cleaer, and cleaner:
I have a data object from Sequelize that is sent by node-express:
{
"page": 0,
"limit": 10,
"total": 4,
"data": [
{
"id": 1,
"title": "movies",
"isActive": true,
"createdAt": "2020-05-30T19:26:04.000Z",
"updatedAt": "2020-05-30T19:26:04.000Z",
"questions": [
{
"questionsCount": 4
}
]
}
]
}
The BIG question is, how do I get the value of questionsCount?
The PROBLEM is, I just can't extract it, these two methods give me undefined result:
category.questions[0].questionsCount
category.questions[0]['questionsCount']
I WAS ABLE to get it using toJSON() (From Sequelize lib I think), like so:
category.questions[0].toJSON().questionsCount
But I'd like to know the answer to the question, or at least a clear explanation of why do I have to use toJSON() just to get the questionsCount?
More context:
I have this GET in my controller:
exports.getCategories = (req, res) => {
const page = myUtil.parser.tryParseInt(req.query.page, 0)
const limit = myUtil.parser.tryParseInt(req.query.limit, 10)
db.Category.findAndCountAll({
where: {},
include: [
{
model: db.Question,
as: "questions",
attributes: [[db.Sequelize.fn('COUNT', 'id'), 'questionsCount']]
}
],
offset: limit * page,
limit: limit,
order: [["id", "ASC"]],
})
.then(data => {
data.rows.forEach(function(category) {
console.log("------ May 31 ----> " + JSON.stringify(category.questions[0]) + " -->" + category.questions[0].hasOwnProperty('questionsCount'))
console.log(JSON.stringify(category))
console.log(category.questions[0].toJSON().questionsCount)
})
res.json(myUtil.response.paging(data, page, limit))
})
.catch(err => {
console.log("Error get categories: " + err.message)
res.status(500).send({
message: "An error has occured while retrieving data."
})
})
}
I loop through the data.rows to get each category object.
The console.log outputs are:
------ May 31 ----> {"questionsCount":4} -->false
{"id":1,"title":"movies","isActive":true,"createdAt":"2020-05-30T19:26:04.000Z","updatedAt":"2020-05-30T19:26:04.000Z","questions":[{"questionsCount":4}]}
4
https://github.com/sequelize/sequelize/blob/master/docs/manual/core-concepts/model-querying-finders.md
By default, the results of all finder methods are instances of the model class (as opposed to being just plain JavaScript objects). This means that after the database returns the results, Sequelize automatically wraps everything in proper instance objects. In a few cases, when there are too many results, this wrapping can be inefficient. To disable this wrapping and receive a plain response instead, pass { raw: true } as an option to the finder method.
(emphasis by me)
Or directly in the source code, https://github.com/sequelize/sequelize/blob/59b8a7bfa018b94ccfa6e30e1040de91d1e3d3dd/lib/model.js#L2028
#returns {Promise<{count: number, rows: Model[]}>}
So the thing is that you get an array of Model objects which you could navigate with their get() method. It's an unfortunate coincidence that you expected an array, and got an array so you thought it is "that" array. Try the {raw:true} thing, I guess it looks something like this:
db.Category.findAndCountAll({
where: {},
include: [
{
model: db.Question,
as: "questions",
attributes: [[db.Sequelize.fn('COUNT', 'id'), 'questionsCount']]
}
],
offset: limit * page,
limit: limit,
order: [["id", "ASC"]],
raw: true // <--- hopefully it is this simple
}) [...]
toJSON() is nearby too, https://github.com/sequelize/sequelize/blob/59b8a7bfa018b94ccfa6e30e1040de91d1e3d3dd/lib/model.js#L4341
/**
* Convert the instance to a JSON representation.
* Proxies to calling `get` with no keys.
* This means get all values gotten from the DB, and apply all custom getters.
*
* #see
* {#link Model#get}
*
* #returns {object}
*/
toJSON() {
return _.cloneDeep(
this.get({
plain: true
})
);
}
So it worked exactly because it did what you needed, removed the get() stuff and provided an actual JavaScript object matching your structure (POJSO? - sorry, I could not resist). I rarely use it and thus always forget, but the key background "trick" is that a bit contrary to its name, toJSON() is not expected to create the actual JSON string, but to provide a replacement object which still gets stringified by JSON.stringify(). (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#toJSON_behavior)
try to do so category.data[0].questions.questionCount
As mentioned by others already, you need category.data[0].questions[0].questionCount.
Let me add to that by showing you why. Look at your object, I annotated it with how each part would be accessed:
category = { // category
"page": 0,
"limit": 10,
"total": 2,
"data": [ // category.data
{ // category.data[0]
"id": 1,
"title": "movies",
"createdAt": "2020-05-30T19:26:04.000Z",
"updatedAt": "2020-05-30T19:26:04.000Z",
"questions": [ // category.data[0].questions
{ // category.data[0].questions[0]
"questionCount": 2 // category.data[0].questions[0].questionCount
}
],
"questionsCount": "newValue here!"
}
]
}
try this
category.data[0].questions[0].questionCount
the reason why you have to use toJSON is because it's sometimes it is used to customise the stringification behavior. like doing some calculation before assinging the value to the object that will be returned , so it is most likley been used here to calculate the "numb of questions and then return an object with the property questionscount and the number calculated
so the object you retreived more or less looks like this
var cathegory = {
data: 'data',
questions:[{
// some calulation here to get the questionsCount
result=4,
toJSON () {
return {"questionsCount":this.result}
}
}
]
};
console.log(cathegory.questions[0].toJSON().questionsCount) //4
console.log(JSON.stringify(cathegory)) // {"data":"data","questions":[{"questionsCount":4}]}
console.log("------ May 31 ----> " + JSON.stringify(cathegory.questions[0]) + " -->" + cathegory.questions[0].hasOwnProperty('questionsCount')) //false
I have been trying to figure out how to do 2fa with webauthn and I have the registration part working. The details are really poorly documented, especially all of the encoding payloads in javascript. I am able to register a device to a user, but I am not able to authenticate with that device. For reference, I'm using these resources:
https://github.com/cedarcode/webauthn-ruby
https://www.passwordless.dev/js/mfa.register.js
And specifically, for authentication, I'm trying to mimic this js functionality:
https://www.passwordless.dev/js/mfa.register.js
In my user model, I have a webauthn_id, and several u2f devices, each of which has a public_key and a webauthn_id.
In my Rails app, I do:
options = WebAuthn::Credential.options_for_get(allow: :webauthn_id)
session[:webauthn_options] = options
In my javascript, I try to mimic the js file above and I do (this is embedded ruby):
options = <%= raw #options.as_json.to_json %>
options.challenge = WebAuthnHelpers.coerceToArrayBuffer(options.challenge);
options.allowCredentials = options.allowCredentials.map((c) => {
c.id = WebAuthnHelpers.coerceToArrayBuffer(c.id);
return c;
});
navigator.credentials.get({ "publicKey": options }).then(function (credentialInfoAssertion)
{
// send assertion response back to the server
// to proceed with the control of the credential
alert('here');
}).catch(function (err)
{
debugger
console.error(err); /* THIS IS WHERE THE ERROR IS THROWN */
});
The problem is, I cannot get past navigator.credentials.get, I get this error in the javascript console:
TypeError: CredentialsContainer.get: Element of 'allowCredentials' member of PublicKeyCredentialRequestOptions can't be converted to a dictionary
options at the time navigator.credentials.get is called looks like this:
I've tried every which way to convert my db-stored user and device variables into javascript properly encoded and parsed variables but cannot seem to get it to work. Anything obvious about what I'm doing wrong?
Thanks for any help,
Kevin
UPDATE -
Adding options json generated by the server:
"{\"challenge\":\"SSDYi4I7kRWt5wc5KjuAvgJ3dsQhjy7IPOJ0hvR5tMg\",\"timeout\":120000,\"allowCredentials\":[{\"type\":\"public-key\",\"id\":\"OUckfxGNLGGASUfGiX-1_8FzehlXh3fKvJ98tm59mVukJkKb_CGk1avnorL4sQQASVO9aGqmgn01jf629Jt0Z0SmBpDKd9sL1T5Z9loDrkLTTCIzrIRqhwPC6yrkfBFi\"},{\"type\":\"public-key\",\"id\":\"Fj5T-WPmEMTz139mY-Vo0DTfsNmjwy_mUx6jn5rUEPx-LsY51mxNYidprJ39_cHeAOieg-W12X47iJm42K0Tsixj4_Fl6KjdgYoxQtEYsNF-LPhwtoKwYsy1hZgVojp3\"}]}"
This is an example of the serialised JSON data returned by our implementation:
{
"challenge": "MQ1S8MBSU0M2kiJqJD8wnQ",
"timeout": 60000,
"rpId": "identity.acme.com",
"allowCredentials": [
{
"type": "public-key",
"id": "k5Ti8dLdko1GANsBT-_NZ5L_-8j_8TnoNOYe8mUcs4o",
"transports": [
"internal"
]
},
{
"type": "public-key",
"id": "LAqkKEO99XPCQ7fsUa3stz7K76A_mE5dQwX4S3QS6jdbI9ttSn9Hu37BA31JUGXqgyhTtskL5obe6uZxitbIfA",
"transports": [
"usb"
]
},
{
"type": "public-key",
"id": "nbN3S08Wv2GElRsW9AmK70J1INEpwIywQcOl6rp_DWLm4mcQiH96TmAXSrZRHciZBENVB9rJdE94HPHbeVjtZg",
"transports": [
"usb"
]
}
],
"userVerification": "discouraged",
"extensions": {
"txAuthSimple": "Sign in to your ACME account",
"exts": true,
"uvi": true,
"loc": true,
"uvm": true
}
}
This is parsed to an object and the code used to coerce those base64url encoded values is:
credentialRequestOptions.challenge = WebAuthnHelpers.coerceToArrayBuffer(credentialRequestOptions.challenge);
credentialRequestOptions.allowCredentials = credentialRequestOptions.allowCredentials.map((c) => {
c.id = WebAuthnHelpers.coerceToArrayBuffer(c.id);
return c;
});
Hope that helps. The JSON data is retreived via a fetch() call and the byte[] fields are encoded as base64url on the serverside.
I've got and app that takes quote input (purity, weight, total), and it pushes to $scope.quote:
// Controller action //
$scope.quote.push({
total: ((($scope.karat * $scope.spot) * $scope.percentage) / 20) * $scope.estimatedWeight,
karat: $scope.karat * 100,
description: $scope.description,
actualWeight: $scope.actualWeight,
estimatedWeight: $scope.estimatedWeight,
percent: $scope.percentage * 100,
spot: $scope.spot
})
and
// Factory //
app.factory('quoteFactory', function() {
var quote = [];
var factory = {};
factory.getQuote = function () {
return quote;
};
return factory;
})
and the post/save upon quote completion
$scope.save = function() {
var now = $scope.getDate();
$scope.quote.push({
createdOn: $scope.getDate()
})
Restangular.all('quote').post($scope.quote).then(function(quote){
$location.path('#/scrap')
});
};
When trying to access the quote JSON for list or edit I can't access all the information needed because of the JSON structure.
{
"0": {
"total": 401.79040000000003,
"karat": 74,
"description": "Rings",
"actualWeight": 12,
"estimatedWeight": 11,
"percent": 80,
"spot": 1234
},
"1": {
"total": 560.7296,
"karat": 56.8,
"description": "Test",
"actualWeight": 22,
"estimatedWeight": 20,
"percent": 80,
"spot": 1234
},
"2": {
"total": 48.5625,
"karat": 92.5,
"description": "Testing",
"actualWeight": 80,
"estimatedWeight": 75,
"percent": 70,
"spot": 20
},
"3": {
"createdOn": "2013-11-26T21:26:42.253Z"
},
"_id": {
"$oid": "52951213e4b05f03172f14e7"
}
}
Each index represents a line item of the quote and the createdOn info. What I'm trying to figure out is if there is a way to be able to access all the line item information without having to call each individual index?
I've looked into some lodash/underscore, thought about restructuring the backend... Not really sure where to go from here.
Complete project code at github
Since there is not much details available as to how your backend works or is handling the data I am assuming the problem is with generating the JSON.
What I would do is change the way Angular posts the data to the server, for example:
the Save function:
$scope.save = function() {
var url = 'http:127.0.0.1:3000/url/you/send/your/data/to',
json = JSON.stringify($scope.quote);
$http({
method: 'POST',
url: url,
data: json,
headers: {
'Content-Type': 'application/json'
}
}).then(function(response) {
$location.path('#/scrap');
});
};
This will give you more control, and will make sure you control how the data gets turned into proper JSON, in this case using the .stringify() method of the JSON object.
I see from your comments that you are concerned with indexing; conventionally objects that sync with the server have a property id that represents their database id's NOT their client array index. This is a good way to keep the model data on the client a representation of the truth on your server side.
There are quite some libraries out there that help with this process client side, I would suggest - since you are using Angular already - reading into Angular Resource.
NOTE: if you want to support older browsers with the JSON.stringify() method, make sure you use Crockford's JSON2.js