Logstash convert entry // JSON - javascript

when I send a http Post to logstach my entry seems sth like that:
{
"_index": "logstash-2014.11.25",
"_type": "logs",
"_id": "PyALYOSjRi-N8cfg8axyMg",
"_score": null,
"_source": {
"message": "technology=apache",
"#version": "1",
"#timestamp": "2014-11-25T11:57:56.931Z",
"host": "127.0.0.1"
},
"sort": [
1416916676931,
1416916676931
]
}
Are you have any idea to handle the message so that it seems like thsi:
{
"_index": "logstash-2014.11.25",
"_type": "logs",
"_id": "PyALYOSjRi-N8cfg8axyMg",
"_score": null,
"_source": {
"#version": "1",
"#timestamp": "2014-11-25T11:57:56.931Z",
"host": "127.0.0.1"
},
"sort": [
1416916676931,
1416916676931
]
**"technology": "apache"**
}

Related

It's possible to find a link in a object if I know how it starts?

I'm getting a response from an API, I'm interested only in one of the link that returns. I have been using for a while
["data"]["tasks"]["0"]["result"]["files"]["0"]["url"]
to get the URL but it seems the API has changed and now the "result" it changes the position 0 in every call...
{"data": {
"id": "dcd99049-63b2",
"tag": "jobbuilder",
"status": "finished",
"created_at": "2022-11-22T21:57:28+00:00",
"started_at": "2022-11-22T21:57:28+00:00",
"ended_at": "2022-11-22T21:57:33+00:00",
"tasks": [
{
"id": "48c34a03-54f2",
"name": "task-2",
"job_id": "dcd99049-63b2",
"status": "finished",
"credits": 0,
"code": null,
"message": null,
"percent": 100,
"operation": "convert",
"engine": "qpdf",
"engine_version": "10.1.0",
"result": {
"files": [
{
"filename": "something.pdf",
"size": 633487
}
]
},
"created_at": "2022-11-22T21:57:30+00:00",
"started_at": "2022-11-22T21:57:32+00:00",
"ended_at": "2022-11-22T21:57:32+00:00",
"retry_of_task_id": null,
"copy_of_task_id": "dd64385a-a2e9",
"user_id": 6070,
"priority": 10,
"host_name": "meggie",
"storage": null,
"depends_on_task_ids": [
"dd64385a-a2e9",
"acfd2a2c-3c45"
],
"links": {
"self": "https://api.cloudconvert.com/v2/tasks/48c34a03-54f2"
}
},
{
"id": "6333e3b8-2f6f",
"name": "export-1",
"job_id": "dcd99049-63b2",
"status": "finished",
"credits": 0,
"code": null,
"message": null,
"percent": 100,
"operation": "export/url",
"result": {
"files": [
{
"filename": "something.pdf",
"size": 633487,
"url": "https://storage.cloudconvert.com/tasks/somethinglink.pdf"
}
]
},
"created_at": "2022-11-22T21:57:28+00:00",
"started_at": "2022-11-22T21:57:32+00:00",
"ended_at": "2022-11-22T21:57:32+00:00",
"retry_of_task_id": null,
"copy_of_task_id": null,
"user_id": 607,
"priority": 10,
"host_name": "meggie",
"storage": "ceph-fra",
"depends_on_task_ids": [
"48c34a03-54f2"
],
"links": {
"self": "https://api.cloudconvert.com/v2/tasks/6333e3b8-2f6f"
}
},
{
"id": "dd64385a-a2e9",
"name": "task-2-create-watermark",
"job_id": "dcd99049-63b2",
"status": "finished",
"credits": 1,
"code": null,
"message": null,
"percent": 100,
"operation": "watermark",
"engine": "imagemagick",
"engine_version": "7.1.0",
"result": {
"files": [
{
"filename": "watermark.pdf",
"size": 609229
}
]
},
"created_at": "2022-11-22T21:57:28+00:00",
"started_at": "2022-11-22T21:57:30+00:00",
"ended_at": "2022-11-22T21:57:32+00:00",
"retry_of_task_id": null,
"copy_of_task_id": null,
"user_id": 6070,
"priority": 10,
"host_name": "meggie",
"storage": "ceph-fra",
"depends_on_task_ids": [
"acfd2a2c-3c45-4e47"
],
"links": {
"self": "https://api.cloudconvert.com/v2/tasks/dd64385a-a2e9"
}
},
{
"id": "acfd2a2c-3c45-4e47",
"name": "task-1",
"job_id": "dcd99049-63b2",
"status": "finished",
"credits": 1,
"code": null,
"message": null,
"percent": 100,
"operation": "capture-website",
"engine": "wkhtml",
"engine_version": "0.12.5",
"result": {
"files": [
{
"filename": "something.pdf",
"size": 27330
}
]
},
"created_at": "2022-11-22T21:57:28+00:00",
"started_at": "2022-11-22T21:57:28+00:00",
"ended_at": "2022-11-22T21:57:30+00:00",
"retry_of_task_id": null,
"copy_of_task_id": null,
"user_id": 6070,
"priority": 10,
"host_name": "meggie",
"storage": null,
"depends_on_task_ids": [],
"links": {
"self": "https://api.cloudconvert.com/v2/tasks/acfd2a2c-3c45"
}
}
],
"links": {
"self": "https://api.cloudconvert.com/v2/jobs/dcd99049-63b2-443e"
}
}};
I would like to always extract only the link:
https://storage.cloudconvert.com/tasks/somethinglink.pdf
which position can change as mentioned before... The link always start with https://storage.cloudconvert.com
I have been using this for a while to access the first object in the array
["data"]["tasks"]["0"]["result"]["files"]["0"]["url"]
but it doesn't seem to work if the link it's in a different position...
This code extracts all values as strings, then finds the one with the necessary prefix:
const data = {"data":{"id":"dcd99049-63b2","tag":"jobbuilder","status":"finished","created_at":"2022-11-22T21:57:28+00:00","started_at":"2022-11-22T21:57:28+00:00","ended_at":"2022-11-22T21:57:33+00:00","tasks":[{"id":"48c34a03-54f2","name":"task-2","job_id":"dcd99049-63b2","status":"finished","credits":0,"code":null,"message":null,"percent":100,"operation":"convert","engine":"qpdf","engine_version":"10.1.0","result":{"files":[{"filename":"something.pdf","size":633487}]},"created_at":"2022-11-22T21:57:30+00:00","started_at":"2022-11-22T21:57:32+00:00","ended_at":"2022-11-22T21:57:32+00:00","retry_of_task_id":null,"copy_of_task_id":"dd64385a-a2e9","user_id":6070,"priority":10,"host_name":"meggie","storage":null,"depends_on_task_ids":["dd64385a-a2e9","acfd2a2c-3c45"],"links":{"self":"https://api.cloudconvert.com/v2/tasks/48c34a03-54f2"}},{"id":"6333e3b8-2f6f","name":"export-1","job_id":"dcd99049-63b2","status":"finished","credits":0,"code":null,"message":null,"percent":100,"operation":"export/url","result":{"files":[{"filename":"something.pdf","size":633487,"url":"https://storage.cloudconvert.com/tasks/somethinglink.pdf"}]},"created_at":"2022-11-22T21:57:28+00:00","started_at":"2022-11-22T21:57:32+00:00","ended_at":"2022-11-22T21:57:32+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":607,"priority":10,"host_name":"meggie","storage":"ceph-fra","depends_on_task_ids":["48c34a03-54f2"],"links":{"self":"https://api.cloudconvert.com/v2/tasks/6333e3b8-2f6f"}},{"id":"dd64385a-a2e9","name":"task-2-create-watermark","job_id":"dcd99049-63b2","status":"finished","credits":1,"code":null,"message":null,"percent":100,"operation":"watermark","engine":"imagemagick","engine_version":"7.1.0","result":{"files":[{"filename":"watermark.pdf","size":609229}]},"created_at":"2022-11-22T21:57:28+00:00","started_at":"2022-11-22T21:57:30+00:00","ended_at":"2022-11-22T21:57:32+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":6070,"priority":10,"host_name":"meggie","storage":"ceph-fra","depends_on_task_ids":["acfd2a2c-3c45-4e47"],"links":{"self":"https://api.cloudconvert.com/v2/tasks/dd64385a-a2e9"}},{"id":"acfd2a2c-3c45-4e47","name":"task-1","job_id":"dcd99049-63b2","status":"finished","credits":1,"code":null,"message":null,"percent":100,"operation":"capture-website","engine":"wkhtml","engine_version":"0.12.5","result":{"files":[{"filename":"something.pdf","size":27330}]},"created_at":"2022-11-22T21:57:28+00:00","started_at":"2022-11-22T21:57:28+00:00","ended_at":"2022-11-22T21:57:30+00:00","retry_of_task_id":null,"copy_of_task_id":null,"user_id":6070,"priority":10,"host_name":"meggie","storage":null,"depends_on_task_ids":[],"links":{"self":"https://api.cloudconvert.com/v2/tasks/acfd2a2c-3c45"}}],"links":{"self":"https://api.cloudconvert.com/v2/jobs/dcd99049-63b2-443e"}}}
const f=i=>i instanceof Object?Object.values(i).flatMap(v=>f(v)):[i+'']
console.log(f(data).find(i=>
i.startsWith('https://storage.cloudconvert.com/tasks/')))

Adding attributes to existing json payload using javascript

I am getting json response as below. I want to add "**
schemas":[
"urn:ietf:params:scim:schemas:core:2.0:User", "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
],
"id":"xyz",
**
to each resource how to do this using javascript. Can anyone help.
{
"Resources": [
{
"emails": {
"type": "bobsmith01#company.com",
"value": "Personal"
},
"name": {
"familyName": "Bob-update3",
"givenName": "Smith"
},
"detail": "SUCCESS",
"userName": "bobsmith01#company.com",
"status": "200"
},
{
"emails": {
"type": "samgomes#company.com",
"value": "Personal"
},
"name": {
"familyName": "gomes",
"givenName": "sam"
},
"detail": "SUCCESS",
"userName": "samgomes#company.com",
"status": "200"
}
]
}
Required output is as below.
{
"Resources": [
{
"schemas":[
"urn:ietf:params:scim:schemas:core:2.0:User",
"urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
],
"id":"xyz",
"emails": {
"type": "bobsmith01#company.com",
"value": "Personal"
},
"name": {
"familyName": "Bob-update3",
"givenName": "Smith"
},
"detail": "SUCCESS",
"userName": "bobsmith01#company.com",
"status": "200"
}
{
"schemas":[
"urn:ietf:params:scim:schemas:core:2.0:User",
"urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
],
"id":"abc",
"emails": {
"type": "samgomes#company.com",
"value": "Personal"
},
"name": {
"familyName": "gomes",
"givenName": "sam"
},
"detail": "SUCCESS",
"userName": "samgomes#company.com",
"status": "200"
}
]
}
You can use ES6 JS spread operator (...) and Array.map to achieve this:
const orgObject = {
"Resources": [{
"emails": {
"type": "bobsmith01#company.com",
"value": "Personal"
},
"name": {
"familyName": "Bob-update3",
"givenName": "Smith"
},
"detail": "SUCCESS",
"userName": "bobsmith01#company.com",
"status": "200"
},
{
"emails": {
"type": "samgomes#company.com",
"value": "Personal"
},
"name": {
"familyName": "gomes",
"givenName": "sam"
},
"detail": "SUCCESS",
"userName": "samgomes#company.com",
"status": "200"
}
]
}
const copiedObject = {...orgObject}; // Copy object to avoid mutation preferably use lodash.cloneDeep
// loop through the object.Resources using map
copiedObject.Resources = copiedObject.Resources.map(res => {
return {
"schemas": [
"urn:ietf:params:scim:schemas:core:2.0:User",
"urn:ietf:params:scim:schemas:extension:enterprise:2.0:User"
],
"id": "xyz",
...res // Use spread operator to copy the existing object properties from Resources
}
});
console.log(copiedObject)
.as-console-wrapper{
top:0;
max-height:100% !important;
}

How to add "collection" field to metadata for Solana NFTs with Visual Studio Code

I'm trying to add the collection field (name and family) to my metadata using VSC. Not sure where to begin or how to do it. Any help appreciated. Thanks!
{
"name": "Solflare X NFT",
"symbol": "",
"description": "Celebratory Solflare NFT for the Solflare X launch",
"seller_fee_basis_points": 0,
"image": "https://www.arweave.net/abcd5678?ext=png",
"animation_url": "https://www.arweave.net/efgh1234?ext=mp4",
"external_url": "https://solflare.com",
"attributes": [
{ "trait_type": "web", "value": "yes" },
{ "trait_type": "mobile", "value": "yes" },
{ "trait_type": "extension", "value": "yes" }
],
"collection": { "name": "Solflare X NFT", "family": "Solflare" },
"properties": {
"files": [
{
"uri": "https://www.arweave.net/abcd5678?ext=png",
"type": "image/png"
},
{
"uri": "https://watch.videodelivery.net/9876jkl",
"type": "unknown",
"cdn": true
},
{ "uri": "https://www.arweave.net/efgh1234?ext=mp4", "type": "video/mp4" }
],
"category": "video",
"creators": [
{ "address": "SOLFLR15asd9d21325bsadythp547912501b", "share": 100 }
]
}
}

How to return an array hits from msearch

I've got an elasticsearch cluster I'm running a search against multiple indices (msearch) which returns an array of objects (one object for each index being queried). Each of these objects has an array of hits inside of a hits object. I'm really only after the _source object. How would you go about getting an array of all the nested "_source" objects?
[
{
"hits": {
"hits": [
{
"_index": "index1",
"_type": "type1",
"_id": "1",
"_score": 12.163426,
"_source": {
"somekey": "some value",
"someotherkey": "another value"
}
}
]
},
},
{
"hits": {
"hits": []
},
},
{
"hits": {
"hits": [
{
"_index": "index2",
"_type": "type2",
"_id": "2",
"_score": 7.0380797,
"_source": {
"somekey": "some value",
"someotherkey": "another value"
}
},
{
"_index": "index2",
"_type": "type2",
"_id": "3",
"_score": 6.07253,
"_source": {
"somekey": "some value 2",
"someotherkey": "another value 2"
}
}
]
},
},
]
You may use Array.prototype.flatMap() to traverse your outer array and Array.prototype.map() to turn hits.hits into array of _source properties as items:
const src = [{"hits":{"hits":[{"_index":"index1","_type":"type1","_id":"1","_score":12.163426,"_source":{"somekey":"some value","someotherkey":"another value"}}]},},{"hits":{"hits":[]},},{"hits":{"hits":[{"_index":"index2","_type":"type2","_id":"2","_score":7.0380797,"_source":{"somekey":"some value1","someotherkey":"another value1"}},{"_index":"index2","_type":"type2","_id":"3","_score":6.07253,"_source":{"somekey":"some value 2","someotherkey":"another value 2"}}]},},],
result = src.flatMap(o => o.hits.hits.map(({_source}) => _source))
console.log(result)
.as-console-wrapper{min-height:100%;}
you can reduce the objects:
const src = [
{
"hits": {
"hits": [
{
"_index": "index1",
"_type": "type1",
"_id": "1",
"_score": 12.163426,
"_source": {
"somekey": "some value",
"someotherkey": "another value"
}
}
]
},
},
{
"hits": {
"hits": []
},
},
{
"hits": {
"hits": [
{
"_index": "index2",
"_type": "type2",
"_id": "2",
"_score": 7.0380797,
"_source": {
"somekey": "some value",
"someotherkey": "another value"
}
},
{
"_index": "index2",
"_type": "type2",
"_id": "3",
"_score": 6.07253,
"_source": {
"somekey": "some value 2",
"someotherkey": "another value 2"
}
}
]
},
},
];
console.log(src.reduce((prev, el) => [...prev, ...el.hits.hits.map(o => o['_source'])], []))

Using aggregate $lookup and $mergeObjects

I want to join collection.
before, I used only lookup, so that I could get separated field that is joined.
but I need to get result similar mysql join.
I noticed there is $lookup and $mergeObjects for this action but not working well.
user collection model.
{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"name": 'admin user',
"email": 'admin#test.com',
"password": 'xxxxxxxx',
"roles": [
{
"id": 0,
"approved": true
},{
"id": 2,
"approved": true
}
]
},{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"name": 'userOne',
"email": 'user#test.com',
"password": 'xxxxxxxx',
"roles": [
{
"id": 1,
"approved": true
}
]
}
roles collection model.
{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"id": '0',
"name": 'administrator'
},{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"id": '0',
"name": 'employeer'
},{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"id": '0',
"name": 'freelancer'
}
after join, I want to get result like below.
{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"name": 'admin user',
"email": 'admin#test.com',
"password": 'xxxxxxxx',
"roles": [
{
"id": 0,
"name": "administrator", //join result
"approved": true
},{
"id": 2,
"name": "freelancer", //join result
"approved": true
}
]
},{
"_id": ObjectId("xxxxxxx"), //this is default id from mongoDB
"name": 'userOne',
"email": 'user#test.com',
"password": 'xxxxxxxx',
"roles": [
{
"id": 1,
"name": "employeer", //join result
"approved": true
}
]
}
You can use below aggregation with mongodb 3.4
You need to $unwind the roles array first and then $group to rollback again
db.users.aggregate([
{ "$unwind": "$roles" },
{ "$lookup": {
"from": "roles",
"localField": "roles.id",
"foreignField": "id",
"as": "roles.role"
}},
{ "$unwind": "$roles.role" },
{ "$addFields": {
"roles": { "$mergeObjects": ["$roles.role", "$roles"] }
}},
{ "$group": {
"_id": "$_id",
"email": { "$first": "$email" },
"password": { "$first": "$password" },
"roles": { "$push": "$roles" }
}},
{ "$project": { "roles.role": 0 }}
])
Which is quite simple with the mongodb 3.6 and above
db.users.aggregate([
{ "$unwind": "$roles" },
{ "$lookup": {
"from": "roles",
"let": { "roleId": "$roles.id", "approved": "$roles.approved" },
"pipeline": [
{ "$match": { "$expr": { "$eq": ["$id", "$$roleId"] }}},
{ "$addFields": { "approved": "$$approved" }}
],
"as": "roles"
}},
{ "$unwind": "$roles" },
{ "$group": {
"_id": "$_id",
"email": { "$first": "$email" },
"password": { "$first": "$password" },
"roles": { "$push": "$roles" }
}}
])
Both will give you similar Output
[
{
"_id": ObjectId("5a934e000102030405000004"),
"email": "user#test.com",
"password": "xxxxxxxx",
"roles": [
{
"_id": ObjectId("5a934e000102030405000001"),
"approved": true,
"id": 1,
"name": "employeer"
}
]
},
{
"_id": ObjectId("5a934e000102030405000003"),
"email": "admin#test.com",
"password": "xxxxxxxx",
"roles": [
{
"_id": ObjectId("5a934e000102030405000000"),
"approved": true,
"id": 0,
"name": "administrator"
},
{
"_id": ObjectId("5a934e000102030405000002"),
"approved": true,
"id": 2,
"name": "freelancer"
}
]
}
]

Categories