ElasticSearch EJS query to fetch missing field - javascript

I have an existing ejs query as below:
let queryBody = ejs.Request()
.size(0)
.query(
ejs.BoolQuery()
.must(
ejs.RangeQuery('hour_time_stamp').gte(this.lastDeviceDate).lte(this.lastDeviceDate)
)
)
.agg(ejs.TermsAggregation('market_agg').field('market').order('sum', 'asc').size(50000)
.agg(ejs.SumAggregation('sum').field('num_devices'))
)
currently the field('market') returns the values where data for market is present. There is data in the database for missing values for market as well, which I need to access. How do I do that?
EDIT:
Values for market in ES is either null or field is missing. I wrote ES query to get all those fields but I am not able to incorporate an ejs query for the same. Any idea how this can be done?
{
"query": {
"bool": {
"should": [
{
"exists": {
"field": "market"
}
},
{
"bool": {
"must_not": [
{
"exists": {
"field": "market"
}
}
]
}
}
]
}
}
}

As per your problem you need a way to group the empty market fields too.
So for that you can use the "missing" value parameter. It defines how the values which are missing(as in your case) are grouped. So you query in json form will be modified like below :-
{
"query":
{
"must": [
"range": {
"hour_time_stamp": {
"gte": lastDeviceDate,
"lte": lastDeviceDate
}
}
]
},
"aggs": {
"market_agg" : {
"market": {
"missing": "empty_markets",
"order": { "sum": "asc" }
}
},
"sum_agg": {
"sum" : { "field" : "num_devices" }
}
}
}
Or in your code it could be done by adding missing parameter like this.
let queryBody = ejs.Request()
.size(0)
.query(
ejs.BoolQuery()
.must(
ejs.RangeQuery('hour_time_stamp').gte(this.lastDeviceDate).lte(this.lastDeviceDate)
)
)
.agg(ejs.TermsAggregation('market_agg').field('market').missing('empty_markets').order('sum', 'asc').size(50000)
.agg(ejs.SumAggregation('sum').field('num_devices'))
)

Related

Automate the writing of complex elastic requests

I get a string from my user that represents an elastic request. There can be the following types of requests:
Single term: TERM:MATCH_TERM ;
Several terms separated by AND: TERM:MATCH_TERM AND TERM:MATCH_TERM AND TERM:MATCH_TERM ;
Several terms separated by OR: TERM:MATCH_TERM OR TERM:MATCH_TERM OR TERM:MATCH_TERM ;
Several terms separated by AND and OR: TERM:MATCH_TERM AND TERM:MATCH_TERM OR TERM:MATCH_TERM AND TERM:MATCH_TERM OR TERM:MATCH_TERM AND TERM:MATCH_TERM ;
The OR operator has priority over the AND operator
I parse this string next and get a list of my different terms.
I want to use this list now to build my elastic query. I know I will have to use the syntax for complex queries: bool. (I use the javascript client of elastic to make my request)
From what I understand, the must filter will allow me to do an AND and the should filter will allow me to do an OR.
In the case where we have: TERM1:MATCH_TERM1 AND TERM2:MATCH_TERM2 OR TERM3:MATCH_TERM3 AND TERM4:MATCH_TERM4 OR TERM5:MATCH_TERM5 AND TERM6:MATCH_TERM6 ;
Would the correct syntax be something like this:
"query":{
"bool":{
"should":[
"must":[
{"terms":{"TERM1":"MATCH_TERM1"}}
] ,
"must":[
{"terms":{"TERM2":"MATCH_TERM2"}}
] ,
],
"should":[
"must":[
{"terms":{"TERM3":"MATCH_TERM3"}}
] ,
"must":[
{"terms":{"TERM4":"MATCH_TERM4"}}
] ,
],
"should":[
"must":[
{"terms":{"TERM5":"MATCH_TERM5"}}
] ,
"must":[
{"terms":{"TERM6":"MATCH_TERM6"}}
] ,
],
}
}
If there is any information missing to answer my question let me know and I will try to add it as soon as possible.
Thank you in advance if you take the time to help me.
{
"query": {
"bool": {
"should": [
{
"bool": {
"filter": [
{
"terms": {
"TERM1": [
"MATCH_TERM1"
]
}
},
{
"terms": {
"TERM2": [
"MATCH_TERM2"
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"TERM3": [
"MATCH_TERM3"
]
}
},
{
"terms": {
"TERM4": [
"MATCH_TERM4"
]
}
}
]
}
},
{
"bool": {
"filter": [
{
"terms": {
"TERM5": [
"MATCH_TERM5"
]
}
},
{
"terms": {
"TERM6": [
"MATCH_TERM6"
]
}
}
]
}
}
],
"minimum_should_match": "1"
}
}
}

How to add a string variable to a JSON query in Javascript without adding extra escaping

I'm having an issues in being able to add a string variable to a section of JSON, and when I pass the full JSON request to the server, the string variable that I added to the JSON always contains extra escape quotes and I don't know how to remove them on insertion.
I've tried using JSON.Parse(queryString) but I then get an exception saying an unexpected token.
The match queries I'm trying to add will vary in number depending on the qty of report filters applied, therefore I can't hard code the match queries in the JSON code block as they're shown in the commented out section.
Code sample:
var queryString = '{ "match": { "log.level": "Information" } }, { "match": { "metadata.log_event_category": "Open Id Connect" } }'
var request =
{
"size": 0,
"query": {
"bool": {
"must": [
// Need to insert a string variable here that would contain the match queries commented out below...
// queryString, // doesnt work, adding the var here results in extra escapes
// JSON.parse(queryString), doesnt work!
// If I manually write in the match queries below then everything goes through OK
//{ "match": { "log.level": "Information" } },
//{ "match": { "metadata.log_event_category": "Open Id Connect" } },
{
"range": {
"#timestamp": {
"gte": chartArray[0].dateTimeFrom,
"lte": chartArray[0].dateTimeTo
}
}
}
]
}
},
"aggs": {
"myAggName": {
"date_histogram": {
"field": "#timestamp",
[elasticIntervalType]: elasticIntervalUnits,
"format": chartArray[0].scaleLabelAttributes
}
},
}
}
Screenshot shows that I'm getting extra escape quotes from the match queries when sending to the server, but I need these to be removed:
Below is a screenshot of a working example but I'm trying to replicate building this equivalent request in javascript in order to pass to my controller and send off to Elastic Search.
It looks like you could just add square brackets around the querystring to make json.parse work. Try:
var queryString = '{ "match": { "log.level": "Information" } }, { "match": { "metadata.log_event_category": "Open Id Connect" } }'
const must = JSON.parse(“[“ + queryString + ”]”);
must.push({
"range": {
"#timestamp": {
"gte": chartArray[0].dateTimeFrom,
"lte": chartArray[0].dateTimeTo
}
}
});
var request =
{
"size": 0,
"query": {
"bool": {
"must": must
}
},
"aggs": { //etc }
};
I had to split the match queries into separate items, then push them individually into the JSON request.
var must = [];
must.push(JSON.parse('{ "match": { "log.level": "Information" } }'))
must.push(JSON.parse('{ "match": { "metadata.log_event_category": "Open Id Connect" } }'))
must.push({
"range": {
"#timestamp": {
"gte": chartArray[0].dateTimeFrom,
"lte": chartArray[0].dateTimeTo
}
}
});
var request =
{
"size": 0,
"query": {
"bool": {
"must": must
}
},
"aggs": {
"myAggName": {
"date_histogram": {
"field": "#timestamp",
[elasticIntervalType]: elasticIntervalUnits,
"format": chartArray[0].scaleLabelAttributes
}
},
}
}

Elasticsearch returns results greater than specified in range

I'm use Elasticsearch (Version: 6.8.4) with MongoDB (4.0.3)
I want find all the documents where price between 1 and 800 and sale_date is greater than Date.now(), but I have problem with my query:
{
"query": {
"bool": {
"must": [],
"filter": {
"term": {
"sold_out": false
},
"bool": {
"should": [
{
"range": {
"sale_date": {
"gt": Date.now()
}
}
},
{
"range": {
"price": {
"gt": 1,
"lte": 800
}
}
}
]
}
}
}
},
"from": 10,
"size": 200
}
It's query returns me results with products where some of them have price greater than 800
Price field stored in Elasticsearch as long
Whant I'm try:
use from: to: and got the same results
change "should" to "must" in filter and it's returns empty results
remove from query { "range": { "sale_date": { "gt": Date.now(), } }
} and it's returns right results!
What I'm doing wrong ?
First remove the uneccessary nesting ofthe two bool objects, try having all the clauses in the same "bool" field like this.
Remeber that with must a document must have the term you are making a comparison on to be included in the result, with should a match will only improve the score, if the condition is met, so in a certain sense must is like AND and should similar to an OR.
In your example changing the filter from should to must made the query return nothing maybe because you don't have any element that has both the date and price you want, or maybe it was the problem was the bool nesting.
Try this:
{
"query": {
"bool": {
"must": [{
"range": {
"sale_date": {
"gt": Date.now()
}
}
},
{
"range": {
"price": {
"gt": 1,
"lte": 800
}
}
}],
"filter": {
"term": {
"sold_out": false
},
}
}
},
"from": 10,
"size": 200
}
This will only get the elements with price lte 800 and sale_date gt Date.now().
Change Date().now() to "gt": "now/d".

Replace parent node by child node in json

I have a JSon object like this:
{
"coach": {
"Errors": {
"items": [
{
"errorMessage": "You must select a date that is in the future.",
"errorBOPath": "twl.date"
}
]
}
}
}
and I want to remove the items but need to move the data of items to Errors, and make Error becomes array. The expected data like this:
{
"coach": {
"Errors": [
{
"errorMessage": "You must select a date that is in the future.",
"errorBOPath": "twl.date"
}
]
}
}
"coach" and "errors" are not fixed, "items" is fixed. eg:
{
"results": {
"items": [
"a",
"b"
]
}
}
Please advice me
jsonObjectVariable.coach.Errors = jsonObjectVariable.coach.Errors.items
https://jsfiddle.net/uq6nc7bm/1/

Graphql: how to query using prefix string

I have the following query in GraphQL:
{
allBibliografia(nome: "Bibliografia 1") {
id
nome
}
}
which works. But I need to query my backend using a prefix string, as the user is typing. For example: if I type "Bibliografia 1" as above, my query returns
{
"data": {
"allBibliografia": [
{
"id": "1",
"nome": "Bibliografia 1"
}
]
}
}
but if I input just "Bibliografia" or "Bibliografia*", I get this
{
"data": {
"allBibliografia": []
}
}
I was hoping that "Bibliografia*" would work just like a wildcard for SQL, but it didn't. Does anybody knows how to achieve that?

Categories