I need to convert a large CSV data set to JSON, however the output should be a JSON dictionary like this:
var products = {
"crystal": {
"description": "This is a crystal",
"price": "2.95"
},
"emerald": {
"description": "This is a emerald",
"price": "5.95"
}
};
This is what the CSV table would look like:
I am using a script referenced here to generate the JSON:
var csv = require('csv')
var fs = require('fs')
var f = fs.createReadStream('Fielding.csv')
var w = fs.createWriteStream('out.txt')
w.write('[');
csv()
.from.stream(f, {columns:true})
.transform(function(row, index) {
return (index === 0 ? '' : ',\n') + JSON.stringify(row);
})
.to.stream(w, {columns: true, end: false})
.on('end', function() {
w.write(']');
w.end();
});
However the output from that script is created in this format:
[
{
"name": "crystal",
"description": "This is a crystal",
"price": "2.95"
},
{
"name": "emerald",
"description": "This is a emerald",
"price": "5.95"
}
]
How would I modify the script to get my desired "dictionary" format?
All you need to do is loop over the array and use item.name as key for your dictionary object
var products ={};
data.forEach(function(item){
products[item.name] = item;
});
This will leave the name property in the item but that shouldn't be an issue
I found csv parser library most useful:
var csvText=`status,path,name,ext,checksum,size,document_service_id,document_service_path,message
success,./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE1.txt,expE1.txt,txt,38441337865069eabae7754b29bb43e1,414984,8269f7e3-3221-49bb-bb5a-5796cf208fd1,/neuroinftest/20170215/expE1.txt,
success,./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE10.txt,expE10.txt,txt,f27e46979035706eb0aaf58c26e09585,368573,2c94ed19-29c9-4660-83cf-c2148c3d6f61,/neuroinftest/20170215/expE10.txt,
success,./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE2.txt,expE2.txt,txt,e1040d9546423c823944120de0e5c46c,333308,b3898f5d-1058-4cf3-acf9-76759117b810,/neuroinftest/20170215/expE2.txt,
`
var csv = require('csv');
csv.parse(csvText, {columns: true}, function(err, data){
console.log(JSON.stringify(data, null, 2));
});
In variable csvText I have my comma-separated file, with the first line serving as a header. I use the parse function and I'm passing the {columns: true} to indicated that the first line has the headers. Second parameter in the callback function (data) has the object with keys being the headers and the values being the corresponding csv cells. I use JSON.stringify to print it nicely and the result object looks like this (it puts it into an array):
[
{
"status": "success",
"path": "./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE1.txt",
"name": "expE1.txt",
"ext": "txt",
"checksum": "38441337865069eabae7754b29bb43e1",
"size": "414984",
"document_service_id": "8269f7e3-3221-49bb-bb5a-5796cf208fd1",
"document_service_path": "/neuroinftest/20170215/expE1.txt",
"message": ""
},
{
"status": "success",
"path": "./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE10.txt",
"name": "expE10.txt",
"ext": "txt",
"checksum": "f27e46979035706eb0aaf58c26e09585",
"size": "368573",
"document_service_id": "2c94ed19-29c9-4660-83cf-c2148c3d6f61",
"document_service_path": "/neuroinftest/20170215/expE10.txt",
"message": ""
},
{
"status": "success",
"path": "./15-02-2017_17-11/d77c7886-ffe9-40f2-b2fe-e68410d07891//expE2.txt",
"name": "expE2.txt",
"ext": "txt",
"checksum": "e1040d9546423c823944120de0e5c46c",
"size": "333308",
"document_service_id": "b3898f5d-1058-4cf3-acf9-76759117b810",
"document_service_path": "/neuroinftest/20170215/expE2.txt",
"message": ""
}
]
UPD: This array can easily be turned into the object you need with reduce:
var res_obj = data.reduce(function(acc, cur, i) {
acc[cur.name] = cur;
return acc;
}, {});
In my case I use the name property as a key. Make sure it's unique.
I think something like this would work :
var products_arr = [{"name":"crystal","description":"This is a crystal","price":"2.95"},
{"name":"emerald","description":"This is a emerald","price":"5.95"}]
var products = {};
for (var i = 0, l = products_arr.length ; i < l ; ++i) {
var x = products_arr[i];
var name = x.name
delete x.name; // deletes name property from JSON object
products[name] = x;
}
This will output :
{
"crystal": {
"description": "This is a crystal",
"price": "2.95"
},
"emerald": {
"description": "This is a emerald",
"price": "5.95"
}
}
If you would like to modify your specific code, you could change the line
return (index === 0 ? '' : ',\n') + JSON.stringify(row);
to
var clonedRow = JSON.parse(JSON.stringify(row));
var key = clonedRow['name'];
delete clonedRow['name'];
var newRow = {};
newRow[key] = clonedRow;
return (index === 0 ? '' : ',\n') + JSON.stringify(newRow);
This creates a new object for each row, modifying the structure according to your requirement.
Your best bet is to use PapaParse, a powerful csv parser/dumper. It supports streams, various string encodings, header row, and it's fast.
Related
I am fetching API into my Express server which has several JSON key value pairs in one array.
For Example:
[{
"quality": "best",
"url": "https://someurlhere.example/?someparameters"
},
{
"quality": "medium",
"url": "https://someurlhere1.example/?someparameters"
}]
And I want to create an array of JSON of that received data in this Format:
[{
"best": "https://someurlhere.example/?someparameters"
},
{
"medium": "https://someurlhere1.example/?someparameters"
}]
I have tried doing this by using for loop
for(let i=0; i < formats.length; i++){
arr.push({
`${formats[i].quality}` : `${formats[i].url}`
})
}
But it didn't work for me.
Please help me in achieving this.
Thanks in Advance :)
You could use the map function and create a new object from it.
For example:
let prevArr = [{
"quality": "best",
"url": "https://someurlhere.example/?someparameters"
}, {
"quality": "medium",
"url": "https://someurlhere1.example/?someparameters"
}]; // Replace with your array
let newArr = [];
let obj = {};
prevArr.map(function(x) {
obj = {};
obj[x["quality"]] = x.url;
newArr.push(obj);
});
const input = [{
"quality": "best",
"url": "https://someurlhere.example/?someparameters"
}, {
"quality": "medium",
"url": "https://someurlhere1.example/?someparameters"
}];
const result = input.map((v, i) => {
return {
[v["quality"]]: v["url"]
}
});
console.log(result)
I am trying to replace / change the values in an object, but I can't seem to work out how it's done or if it can even be done.
I'm trying to add https://SiteName.com to the start of each of the values so it will be like https://SiteName.com\/asset\/image\/map\/map-grass.svg
var assets = [{
"name": "\/asset\/image\/map\/map-grass.svg",
"url": "\/asset\/image\/map\/map-grass.svg"
}, {
"name": "\/asset\/image\/map\/map-stone.svg",
"url": "\/asset\/image\/map\/map-stone.svg"
}]
Object.keys(assets).forEach(key => {
const val = assets[key];
console.log(val)
});
Try this:
var assets = [{
"name": "\/asset\/image\/map\/map-grass.svg",
"url": "\/asset\/image\/map\/map-grass.svg"
}, {
"name": "\/asset\/image\/map\/map-stone.svg",
"url": "\/asset\/image\/map\/map-stone.svg"
}]
let url = "https://SiteName.com";
Object.keys(assets).forEach(key => {
const val = assets[key];
val.name = url + val.name;
val.url = url + val.url;
});
console.log(assets)
You need a nested loop (or forEach) here - one to go over the elements of the assets array, and then, for each object in there, go over all its properties:
var assets = [{
"name": "\/asset\/image\/map\/map-grass.svg",
"url": "\/asset\/image\/map\/map-grass.svg"
}, {
"name": "\/asset\/image\/map\/map-stone.svg",
"url": "\/asset\/image\/map\/map-stone.svg"
}]
assets.forEach(o => {
Object.keys(o).forEach(key => {
o[key] = 'https://SiteName.com' + o[key];
})
});
console.log(assets);
I have this JSON Response from API call
[
{
"id": 20599,
"name": "Deliver",
"options": [
{
"id": 63775,
"name": "Item",
"dataType": "SelectMultiOption",
"required": false,
"options": [
{
"id": 426,
"name": "Towels"
},
{
"id": 427,
"name": "Toothbrush"
},
{
"id": 428,
"name": "Pillow"
}
]
}
]
}
]
I am using this code to get the id of the service "Deliver"
var data = JSON.parse(responseBody);
var loop_count = 0
for (count = 0; count < data.length; count++)
{
if (data[count].name == "Deliver")
{
var job_id = data[count].id;
postman.setEnvironmentVariable("service_id", job_id);
}
}
The questions are:
How can I get value from array "options", I need to get the "id":
63775 and store as "item_id" and the "name":"Item" as "item_name" postman variables.
Then I need to select the "options" nested in record
"Item" and select the option "name": "Toothbrush" and store in postman
variable "svc_optn_optn_name" and it's "id" stored in
"svc_optn_optn_id"
Here I am giving my own suggestion for your problem with few lines of code. I am not sure, how are you going to use these values. I also don't know if the outer options array will always have 1 item or more. I have just tried to satisfy your questions.
Please ask/comment, if you have more doubts or I am wrong.
I have created a function getAllPostmanDataFrom(obj) which takes object as parameter which is the value of data[count], gathers necessary info in other object postmanObj and returns it to the caller.
function getAllPostmanDataFrom(obj) {
const item_id = obj.options[0].id;
const item_name = obj.options[0].name;
const svc_optn_optn_name = obj.options[0].options[1].name;
const svc_optn_optn_id = obj.options[0].options[1].id;
const postmanObj = {item_id, item_name, svc_optn_optn_id, svc_optn_optn_name}; // Return object
return postmanObj;
}
var data = [
{
"id": 20599,
"name": "Deliver",
"options": [
{
"id": 63775,
"name": "Item",
"dataType": "SelectMultiOption",
"required": false,
"options": [
{
"id": 426,
"name": "Towels"
},
{
"id": 427,
"name": "Toothbrush"
},
{
"id": 428,
"name": "Pillow"
}
]
}
]
}
]
var count = 0;
var obj = data[count];
var postmanObj = getAllPostmanDataFrom(obj);
//var {item_id, item_name, svc_optn_optn_id} = postmanObj;
console. log(postmanObj)
/*
console.log(item_id);
console.log(item_name);
console.log(svc_optn_optn_id);
console.log(svc_optn_optn_name);
*/
Finally, you can use values contained in postmanObj as follows:.
postman.setEnvironmentVariable("item_id", postmanObj.item_id);
postman.setEnvironmentVariable("item_name", postmanObj.item_name);
And so on.
This is the solution
var data = JSON.parse(responseBody);
variable named as data
var loop_count = 0
for (count = 0; count < data.length; count++)
{
if (data[count].name == "Deliver")
{
var job_id = data[count].id;
postman.setEnvironmentVariable("service_id", job_id);
var job1_name = data[count].options[0].name;
postman.setEnvironmentVariable("item_name", job1_name);
var job2_id = data[count].options[0].id;
postman.setEnvironmentVariable("item_id", job2_id);
var job3_id = data[count].options[0].options[1].id;
postman.setEnvironmentVariable("svc_optn_optn_id", job3_id);
var job4_name = data[count].options[0].options[1].name;
postman.setEnvironmentVariable("svc_optn_optn_name", job4_name);
}
const data = JSON.parse(responseBody);
data.forEach(item => {
console.log(item.id); // deliver object id.
item.options.forEach(option => {
console.log(`Option Id ${option.id}`); // option id
postman.setEnvironmentVariable("service_id", option.id);
option.options(optionItem => {
if(optionItem.name == 'Toothbrush'){
postman.setEnvironmentVariable("svc_optn_optn_name", optionItem.name);
postman.setEnvironmentVariable("svc_optn_optn_id", optionItem.id);
}
});
});
});
I am creating a Tableau Web Data Connector as described in their Getting Started guide HERE.
I have implemented a similar solution previous with data from a basic associative array, but I am having trouble with my current API call.
I make an API call to an external web service that returns a JSON similar to the one listed below (simplified version)(COMMENT simply added for clarity and not part of original code).
{
"status": true,
"employee": {
"id": 123
},
"company": {
"id": 123
},
"job": {
"id": 123,
"job_workflows": [{
"id": 1,
"name": "Start",
"action_value_entered": "Start"
}, {
"id": 2,
"name": "Date",
"action_value_entered": "2017-09-11"
}, {
"id": 3,
"name": "Crew",
"action_value_entered": "Crew 3"
},
**COMMENT** - 17 other unique fields - omitted for brevity
]
}
}
For my requirements, I need to create a new column for each of my JSON job_workflows to display the data in Tableau. I.e.
Column 1 Header = Start
Column 1 value = Start
Column 2 Header = Date Complete
Column 2 value = 2017-09-11
Etc.
My Tableau JavaScript file looks as below:
(function () {
var myConnector = tableau.makeConnector();
myConnector.init = function(initCallback) {
initCallback();
tableau.submit();
};
myConnector.getSchema = function (schemaCallback) {
var cols = [
{ id : "start", alias : "Start", dataType: tableau.dataTypeEnum.string },
{ id : "date", alias : "Date", dataType: tableau.dataTypeEnum.datetime },
{ id : "crew", alias : "Crew", dataType: tableau.dataTypeEnum.string }
];
var tableInfo = {
id : "myData",
alias : "My Data",
columns : cols
};
schemaCallback([tableInfo]);
};
myConnector.getData = function (table, doneCallback) {
$.getJSON("http://some/api/call/data.php", function(response) {
var resp = response; // Response is JSON as described above
var tableData = [];
// Iterate over the JSON object
for (var i = 0, len = feat.length; i < len; i++) {
// not working
tableData.push({
"start": resp.job.job_workflows[i].action_value_entered,
"date": resp.job.job_workflows[i].action_value_entered,
"crew": resp.job.job_workflows[i].action_value_entered
});
}
table.appendRows(tableData);
doneCallback();
});
};
tableau.registerConnector(myConnector);
})();
How do I iterate over the JSON job_workflow and assign each action_value_entered to be a id in my getSchema() function? My current version simply hangs and no values are returned. NO error or warnings thrown.
In this fiddle attempting to iterate over json String :
http://jsfiddle.net/sfbanvv6/2/
But receive exception :
Uncaught SyntaxError: Unexpected token e
JSON is valid ? So something incorrect with how I'm iterating ?
fiddle code :
javascript :
jQuery(document).ready(function() {
var jsonString2 = '{ "header": "test1", "children": [ { "date": "2015-01-02", "count": "36" }, { "date": "2015-01-03", "count": "29" } ] }'
var data2 = []
var combinedHeights2=[]
var myData2 = JSON.parse(jsonString2);
$.each(myData2, function(i, obj) {
console.log($.parseJSON(obj))
$.each(obj.children, function(i2, obj2) {
console.log('here')
var d = new Date(obj2.date)
console.log(Date.parse(d))
// data2.push([Date.parse(d), parseInt(obj.count)])
// combinedHeights2.push(parseInt(obj.count))
});
});
});
$.parseJSON(obj) is attempting to parse something that is already parsed. JSON parsers don't parse only the top level and leave the inner JSON as JSON, they return an entire nested set of arrays and objects. So your error may be caused by your logging code.
The cure is to recognize that obj doesn't need any parsing and you can just use it directly.
Getting more specific, this code will do what you seem to want:
var jsonString2 = '{ "header": "test1", "children": [
{ "date": "2015-01-02", "count": "36" },
{ "date": "2015-01-03", "count": "29" }
] }';
var data2 = [];
var combinedHeights2 = [];
var myData2 = JSON.parse(jsonString2);
console.log(myData2);
$.each(myData2.children, function(i, obj) {
console.log(obj);
var d = new Date(obj.date);
console.log(Date.parse(d));
});
Especially pay attention to what myData2 looks like when logged. You need to use myData2.children in your each, not myData2 by itself.
P.S. Get some semicolons!
With chrome logging formats you can more clearly see what you are iterating with by changing your logging statement to;
$.each(myData2, function(i, obj) {
console.log('%s %O', i, obj);
});
Your first obj is property in the loop header = 'test1' there is no children property for this. The second is children = [ ] which is your array.
To iterate over the array just use;
$.each(myData2.children, function(i, obj) {
var d = new Date(obj.date)
}
obj.children was undefined. Here its the working one
var myData2 = JSON.parse(jsonString2);
$.each(myData2, function(i, obj) {
var t = myData2.children;
$.each(t, function(i2, obj2) {
console.log(obj2);
//var d = new Date(obj2.date);
//console.log(Date.parse(d));
// data2.push([Date.parse(d), parseInt(obj.count)])
// combinedHeights2.push(parseInt(obj.count))
});
});
but here is a better version to get only dates
$.each(myData2.children, function(i, obj) {
var t = this;
console.log(t.date);
var d = new Date(t.date);
});
});
If you are trying to iterate the children property of your Json then you can do it this way:
jQuery(document).ready(function() {
var jsonString2 = '{ "header": "test1", "children": [ { "date": "2015-01-02", "count": "36" }, { "date": "2015-01-03", "count": "29" } ] }'
var myData2 = JSON.parse(jsonString2);
$.each(myData2.children, function(i, obj) {
console.log(obj);
});
});
<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.0/jquery.min.js"></script>
You were getting the exception because your code tried to iterate over your "header" parameter which holds a String and therefor cannot be iterated.