We Keep Coding

Combine duplicate tokens inside huge JSON file into nested array of objects using React

I looked at several of the suggested solutions but none seemed to rise to this confounding data formatting challenge.
I have a huge JSON file (over 100k rows) and massive duplicates of data all as top level objects. Here's an example:
[
{
"manufacturer":"Samsung",
"device":"Galaxy A32 5G",
"model":"SM-A326B",
"chipset":"Mediatek MT6853V/NZA",
"date":"2022-01-01",
"fw_id":"A326BXXS4AVA1",
"android":"R(Android 11)",
"known_passcode":false,
"afu":false,
"bfu":false,
"bruteforce":false
},
{
"manufacturer":"Samsung",
"device":"Galaxy A32 5G",
"model":"SM-A326U",
"chipset":"Mediatek MT6853V/NZA",
"date":"2021-03-01",
"fw_id":"A326USQU1AUD4",
"android":"R(Android 11)",
"known_passcode":true,
"afu":false,
"bfu":true,
"bruteforce":true
},
{
"manufacturer":"Samsung",
"device":"Galaxy A32 5G",
"model":"SM-A326U1",
"chipset":"Mediatek MT6853V/NZA",
"date":"2021-09-01",
"fw_id":"A326U1UEU5AUJ2",
"android":"R(Android 11)",
"known_passcode":true,
"afu":false,
"bfu":true,
"bruteforce":true
},
{
"manufacturer":"LGE",
"device":"LG K31",
"model":"LGL355DL",
"chipset":"Mediatek MT6762",
"date":"unknown",
"fw_id":"L355DL10l",
"android":"unknown",
"known_passcode":false,
"afu":false,
"bfu":false,
"bruteforce":false
}
]
This needs to be organized so that data points like manufacturer, device, model are not duplicated hundreds of times.
Btw, here's a JSFiddle to play with:
https://jsfiddle.net/xpancom/Lq7duahv/
Ideally, the JSON format would be the following:
[
{
"manufacturers": [
{
"manufacturer": "Samsung",
"devices": [
{
"device": "Galaxy A32 5G",
"models": [
{
"model": "SM-A326B",
"data": [
{
"chipset": "Mediatek MT6853V/NZA",
"date": "2022-01-01",
"fw_id": "A326BXXS4AVA1",
"android": "R(Android 11)",
"known_passcode": false,
"afu": false,
"bfu": false,
"bruteforce": false
},
{
"chipset": "Mediatek MT6853V/NZA",
"date": "2021-09-01",
"fw_id": "A326BXXU3AUH7",
"android": "R(Android 11)",
"known_passcode": true,
"afu": false,
"bfu": true,
"bruteforce": true
}
]
},
{
"model": "SM-A326U1",
"data": [
{
"chipset": "Mediatek MT6853V/NZA",
"date": "2021-09-01",
"fw_id": "A326U1UEU5AUJ2",
"android": "R(Android 11)",
"known_passcode": true,
"afu": false,
"bfu": true,
"bruteforce": true
}
]
}
]
}
]
},
{
"manufacturer": "LGE",
"devices": [
{
"device": "LG K31",
"models": [
{
"model": "SM-A326B",
"data": [
{
"chipset": "Mediatek MT6762",
"date": "unknown",
"fw_id": "L355DL10l",
"android": "unknown",
"known_passcode": false,
"afu": false,
"bfu": false,
"bruteforce": false
}
]
}
]
}
]
}
]
}
]
Working in React, here's what I've got so far in trying to massage this data:
const source = data;
const destination = [];
const classifiedTokens = []; // will be used to stored already classified tokens
const classifiedTokensModel = []; // will be used to stored already classified tokens for models
const getNextTokenArray = (source) => {
let unusedToken = null;
const nextTokenArray = source.filter(function (element) {
if (!unusedToken && !classifiedTokens.includes(element['device'])) {
unusedToken = element['device'];
classifiedTokens.push(unusedToken);
}
return unusedToken ? unusedToken === element['device'] : false;
});
return unusedToken ? nextTokenArray : null;
};
// Pass in arrays deconstructed from addToDestination to process third tier nested objects for models
const getNextTokenArrayModel = (tokenObject) => {
let tokenObjectDevice = tokenObject['device'];
let tokenObjectData = tokenObject['data'];
let unusedTokenModel = null;
const nextTokenArrayModel = tokenObjectData.filter(function (element) {
if (!unusedTokenModel && !classifiedTokensModel.includes(element['model'])) {
unusedTokenModel = element['model'];
classifiedTokensModel.push(unusedTokenModel);
}
return unusedTokenModel ? unusedTokenModel === element['model'] : false;
});
//return unusedTokenModel ? nextTokenArrayModel : null;
if (unusedTokenModel) {
if (nextTokenArrayModel.length === 0) return;
let res = {
device: tokenObjectDevice,
model: nextTokenArrayModel[0]['model'],
data: [],
};
nextTokenArrayModel.forEach((element) => {
res.data.push({
manufacturer: element.manufacturer,
chipset: element.chipset,
date: element.date,
fw_id: element.fw_id,
android: element.android,
knownPasscode: element.knownPasscode,
afu: element.afu,
bfu: element.bfu,
bruteforce: element.bruteforce,
});
});
destination.push(res);
} else {
return null;
}
};
const addToDestination = (tokenArray) => {
if (tokenArray.length === 0) return;
let res = {
device: tokenArray[0]['device'],
data: [],
};
tokenArray.forEach((element) => {
res.data.push({
manufacturer: element.manufacturer,
model: element.model,
chipset: element.chipset,
date: element.date,
fw_id: element.fw_id,
android: element.android,
knownPasscode: element.knownPasscode,
afu: element.afu,
bfu: element.bfu,
bruteforce: element.bruteforce,
});
});
getNextTokenArrayModel(res); // Call this to process and group nested model duplicates by device
//destination.push(res);
};
let nextTokenArray = getNextTokenArray(source);
while (nextTokenArray) {
addToDestination(nextTokenArray);
nextTokenArray = getNextTokenArray(source);
}
setTimeout(() => {
document.getElementById('root').innerHTML =
'<pre>' + JSON.stringify(destination, null, 2) + '</pre>';
}, 1000);
};
And here's the JSFiddle again:
https://jsfiddle.net/xpancom/Lq7duahv/
Who can smash this data formatting dilemma?

This answer is not React specific, but one approach would be to use array.reduce() to transform each level/node of the structure as shown in the code snippet below.
const source = [
{
manufacturer: 'Samsung',
device: 'Galaxy A32 5G',
model: 'SM-A326B',
chipset: 'Mediatek MT6853V/NZA',
date: '2022-01-01',
fw_id: 'A326BXXS4AVA1',
android: 'R(Android 11)',
known_passcode: false,
afu: false,
bfu: false,
bruteforce: false,
},
{
manufacturer: 'Samsung',
device: 'Galaxy A32 5G',
model: 'SM-A326B',
chipset: 'Mediatek MT6853V/NZA',
date: '2022-01-01',
fw_id: 'A326BXXS4AVA1',
android: 'R(Android 11)',
known_passcode: false,
afu: false,
bfu: false,
bruteforce: false,
},
{
manufacturer: 'Samsung',
device: 'Galaxy A32 5G',
model: 'SM-A326U',
chipset: 'Mediatek MT6853V/NZA',
date: '2021-03-01',
fw_id: 'A326USQU1AUD4',
android: 'R(Android 11)',
known_passcode: true,
afu: false,
bfu: true,
bruteforce: true,
},
{
manufacturer: 'Samsung',
device: 'Galaxy A32 5G',
model: 'SM-A326U1',
chipset: 'Mediatek MT6853V/NZA',
date: '2021-09-01',
fw_id: 'A326U1UEU5AUJ2',
android: 'R(Android 11)',
known_passcode: true,
afu: false,
bfu: true,
bruteforce: true,
},
{
manufacturer: 'LGE',
device: 'LG K31',
model: 'LGL355DL',
chipset: 'Mediatek MT6762',
date: 'unknown',
fw_id: 'L355DL10l',
android: 'unknown',
known_passcode: false,
afu: false,
bfu: false,
bruteforce: false,
},
];
function generateTree(data, key) {
return data.reduce((acc, val) => {
// Split the key name from the child data
const { [key.name]: keyName, ...childData } = val;
// Find a tree item in the structure being generated
const treeItem = acc.find((item) => item[key.name] === keyName);
if (treeItem) {
// If found, append child data
treeItem[key.child].push(childData);
} else {
// If not found, create new key and append child data
acc.push({ [key.name]: keyName, [key.child]: [childData] });
}
return acc;
}, []);
}
// Generate manufacturer/device structure
const manufacturers = generateTree(source, {
name: 'manufacturer', // Key name to use as grouping identifier
child: 'devices', // Key name for child data
});
// Generate device/model structure
manufacturers.forEach((manufacturer) => {
manufacturer.devices = generateTree(manufacturer.devices, {
name: 'device',
child: 'models',
});
// Generate model/data structure
manufacturer.devices.forEach((device) => {
device.models = generateTree(device.models, {
name: 'model',
child: 'data',
});
});
});
const destination = [{ manufacturers }];
console.log(destination);

Show data to Ag Grid Vue

I am trying to load data into the VueJS table using the ag Grid vue plugin that I have in a template, I have tried several ways but I am not getting the result I am looking for ..., getting the following result:
{
"users": [
{
"id": 1,
"client_id": 1,
"cop_id": null,
"role_id": null,
"name": "Bart",
"email": "correo.123#gmail.com",
"created_at": null,
"updated_at": null
}
]
}
then I have my Vue view in this way, and sorry for including all the code, but since the template is working with everything that comes, I prefer to deliver all the detail, I have tried to make changes to the created method with axios, but I did not It works, unfortunately I can't load the data into the table, I hope you can guide me to fix my loading error.
<template>
<div id="page-user-list">
<!-- AgGrid Table -->
<ag-grid-vue
ref="agGridTable"
:components="components"
:gridOptions="gridOptions"
class="ag-theme-material w-100 my-4 ag-grid-table"
:columnDefs="columnDefs"
:defaultColDef="defaultColDef"
:rowData="rowData"
rowSelection="multiple"
colResizeDefault="shift"
:animateRows="true"
:floatingFilter="true"
:pagination="true"
:paginationPageSize="paginationPageSize"
:suppressPaginationPanel="true"
:enableRtl="$vs.rtl">
</ag-grid-vue>
<vs-pagination
:total="totalPages"
:max="10"
v-model="currentPage" />
</div>
</template>
<script>
import { AgGridVue } from "ag-grid-vue"
import '#sass/vuexy/extraComponents/agGridStyleOverride.scss'
import vSelect from 'vue-select'
import axios from 'axios'
// Store Module
//import moduleUserManagement from '#/store/user-management/moduleUserManagement.js'
// Cell Renderer
import CellRendererLink from "./cell-renderer/CellRendererLink.vue"
import CellRendererStatus from "./cell-renderer/CellRendererStatus.vue"
import CellRendererVerified from "./cell-renderer/CellRendererVerified.vue"
import CellRendererActions from "./cell-renderer/CellRendererActions.vue"
export default {
components: {
AgGridVue,
vSelect,
// Cell Renderer
CellRendererLink,
CellRendererStatus,
CellRendererVerified,
CellRendererActions,
},
data() {
return {
// Filter Options
roleFilter: { label: 'Todos', value: 'all' },
roleOptions: [
{ label: 'Todos', value: 'all' },
{ label: 'Administador', value: 'admin' },
{ label: 'Usuario', value: 'user' },
{ label: 'Staff', value: 'staff' },
],
statusFilter: { label: 'Todos', value: 'all' },
statusOptions: [
{ label: 'Todos', value: 'all' },
{ label: 'Activo', value: 'active' },
{ label: 'Desactivado', value: 'deactivated' },
{ label: 'Bloqueado', value: 'blocked' },
],
isVerifiedFilter: { label: 'Todos', value: 'all' },
isVerifiedOptions: [
{ label: 'Todos', value: 'all' },
{ label: 'Si', value: 'yes' },
{ label: 'No', value: 'no' },
],
departmentFilter: { label: 'Todos', value: 'all' },
departmentOptions: [
{ label: 'Todos', value: 'all' },
{ label: 'Vendido', value: 'sales' },
{ label: 'Departamento', value: 'development' },
{ label: 'Administrado', value: 'management' },
],
searchQuery: "",
// AgGrid
gridApi: null,
gridOptions: {},
defaultColDef: {
sortable: true,
resizable: true,
suppressMenu: true
},
columnDefs: [
{headerName: 'ID', field: 'id', width: 125, filter: true, checkboxSelection: true, headerCheckboxSelectionFilteredOnly: true, headerCheckboxSelection: true },
{headerName: 'Username', field: 'client_id', filter: true, width: 210, cellRendererFramework: 'CellRendererLink'},
{headerName: 'Email', field: 'email', filter: true, width: 225 },
{headerName: 'Nombre', field: 'name', filter: true, width: 200 },
{headerName: 'Comuna', field: 'cop_id', filter: true, width: 150},
{headerName: 'Role', field: 'role_id', filter: true, width: 150},
{headerName: 'Acciones', width: 150, cellRendererFramework: 'CellRendererActions'},
],
// Cell Renderer Components
components: {
CellRendererLink,
CellRendererStatus,
CellRendererVerified,
CellRendererActions,
}
}
},
watch: {
roleFilter(obj) {
this.setColumnFilter("role", obj.value)
},
statusFilter(obj) {
this.setColumnFilter("status", obj.value)
},
isVerifiedFilter(obj) {
let val = obj.value === "all" ? "all" : obj.value == "yes" ? "true" : "false"
this.setColumnFilter("is_verified", val)
},
departmentFilter(obj) {
this.setColumnFilter("department", obj.value)
},
},
computed: {
usersData() {
return this.users.data
},
paginationPageSize() {
if(this.gridApi) return this.gridApi.paginationGetPageSize()
else return 10
},
totalPages() {
if(this.gridApi) return this.gridApi.paginationGetTotalPages()
else return 0
},
currentPage: {
get() {
if(this.gridApi) return this.gridApi.paginationGetCurrentPage() + 1
else return 1
},
set(val) {
this.gridApi.paginationGoToPage(val - 1)
}
}
},
methods: {
setColumnFilter(column, val) {
const filter = this.gridApi.getFilterInstance(column)
let modelObj = null
if(val !== "all") {
modelObj = { type: "equals", filter: val }
}
filter.setModel(modelObj)
this.gridApi.onFilterChanged()
},
resetColFilters() {
// Reset Grid Filter
this.gridApi.setFilterModel(null)
this.gridApi.onFilterChanged()
// Reset Filter Options
this.roleFilter = this.statusFilter = this.isVerifiedFilter = this.departmentFilter = { label: 'All', value: 'all' }
this.$refs.filterCard.removeRefreshAnimation()
},
updateSearchQuery(val) {
this.gridApi.setQuickFilter(val)
}
},
created(){
//let users = /apps/user/user-list/
axios.get('api/users')
.then(res => {
let users = res.data.users;
})
}
}
</script>
<style lang="scss">
#page-user-list {
.user-list-filters {
.vs__actions {
position: absolute;
right: 0;
top: 50%;
transform: translateY(-58%);
}
}
}
</style>

Creating An Association between 2 models In Express-Sequelize MySql

Disclaimer: I am very new to
Node/Express/Sequelize
Questions:
1. Do I need to import visitors.js to visitorsInfo.js so that I can create an association between the 2?
2. If not, how do I set up the visitorsInfo_id as a Foreign Key from visitors.js column visitors_id?
Snippet:
...model/visitors.js
'use strict'
module.exports = ( sequelize , type ) => {
return sequelize.define( 'visitors' , {
visitor_id: {
type: type.INTEGER,
primaryKey: true,
autoIncrement: true
},
web_status: {
type: type.BOOLEAN
},
digital_status: {
type: type.BOOLEAN
},
hosting_status: {
type: type.BOOLEAN
},
training_status: {
type: type.BOOLEAN
},
})
}
.../model/visitors_info.js
'use strict'
module.exports = ( sequelize , type) => {
return sequelize.define( 'user_info' , {
visitorsInfo_id: {
type: type.INTEGER,
/*
How to set up foreign key...?
*/
},
firstname: {
type: type.STRING
},
lastname: {
type: type.STRING
},
company: {
type: type.STRING
},
contact_info: {
type: type.INTEGER
}
})
}

No need import visitors.js to visitorsInfo.js
Base on the document from Sequelize, In file visitorsInfo.js
'use strict'
module.exports = ( sequelize , type) => {
var user_info = sequelize.define( 'user_info' , {
visitorsInfo_id: {
type: type.INTEGER,
},
firstname: {
type: type.STRING
},
lastname: {
type: type.STRING
},
company: {
type: type.STRING
},
contact_info: {
type: type.INTEGER
}
});
user_info.associate = function (models) {
// associations can be defined here
user_info.belongsTo(models.visitors, {
as: 'visitors',
foreignKey: 'visitorsInfo_id',
targetKey: 'visitor_id'
});
}
return user_info
}

Javascript `this` changes constantly, don't know why

Currently, I'm writing an app on Node.js 5.2.0 on a Linux box with Redis and Caminte. When trying to add different prototype methods to a database object, the context of what this refers to constantly shifts within our reference. After calling push in modelRules.js, this shifts types. I was looking for some assistance with:
How to consistently reference the instantiation of a specific module (function that accepts a schema object) outside of the module itself. I want to tack on prototype functions such as addModelBelongsTo to a User object, and sadly my function simply seems to break when referencing the internal modifiable data members within the class.
The proper organization of the prototype accessors. Is there a specific style that should be used when referencing the insides of the instantiations of these classes?
Why the instantiation of the class User persists data across multiple instantiations of the class? For self[restructuredModelName] (type of array), whenever I call this method on one instantiation, another instantiation of the other object already contains the data of the first instantiation. This should not be happening.
User.js
module.exports = function (schema) {
const IBANVerificationStatusSymbol = Symbol('IBANVerificationStatus');
const relationalMapper = require('./../relationalMapper');
const userObj = {
id: { type: schema.Number },
firstName: { type: schema.String },
lastName: { type: schema.String },
IBAN: { type: schema.String, unique: true },
IBANVerified: { type: schema.Boolean, default: false },
IBANVerificationCode: { type: schema.String },
BIC: { type: schema.String },
email: { type: schema.String, index: true, unique: true },
password: { type: schema.String },
status: { type: schema.Number, default: 0 },
profilePicture: { type: schema.String },
phone: { type: schema.String, index: true, unique: true },
accessToken: { type: schema.String },
prefix: { type: schema.String, default: '+31' },
updated: { type: schema.Date, default: Date.now() },
created: { type: schema.Date, default: Date.now() },
lastAccessedFeed: { type: schema.Date },
street: { type: schema.String },
streetNumber: { type: schema.String },
postalCode: { type: schema.String },
city: { type: schema.String },
country: { type: schema.String },
FCR: { type: schema.Number, default: 0 },
};
// There's GOTTA be a way to typecheck at compilation
const associationMap = {
Activity: { belongsTo: 'Activity', hasMany: 'activities' },
Client: { belongsTo: null, hasMany: 'clients' },
Group: { belongsTo: 'Group', hasMany: 'groups' },
Mandate: { belongsTo: null, hasMany: 'mandates' },
Transaction: { belongsTo: null, hasMany: 'transactions' },
Update: { belongsTo: null, hasMany: 'updates' },
Reaction: { belongsTo: null, hasMany: 'reactions' },
};
relationalMapper.createRelations(associationMap, userObj, schema);
const User = schema.define('user', userObj, {
});
const setId = function (self, models) {
// self.addClients(1);
};
User.hooks = {
afterInitialize: [setId],
};
User.prototype.obj = userObj;
User.associationMap = associationMap;
User.prototype.associationMap = associationMap;
return User;
};
modelRules.js:
function addModelBelongsTo(modelName, models, modelObjKey, modelRelated) {
const restructuredModelName = `memberOf${modelName}`;
const restructuredModelNameCamel = `addMemberOf${modelName}`;
const currentModels = models;
currentModels[modelObjKey].prototype[restructuredModelNameCamel] = function(modelId) {
const self = this;
return new Promise((resolve, reject) => {
if (self[restructuredModelName].indexOf(modelId) <= -1) {
modelRelated.exists(modelId, function(err, exists) {
if (err || !exists) { reject(new Error(err || 'Doesnt exist')); }
console.log(`This:${self}\nrestructuredModelName:${JSON.stringify(self[restructuredModelName])}`);
self[restructuredModelName].push(modelId);
console.log(`This:${self}\nrestructuredModelName:${restructuredModelName}`);
self.save((saveErr) => {
saveErr ? reject(new Error(saveErr)) : resolve(self);
});
});
} else {
reject(new Error(''));
}
});
};
}

scraping data from Javascript in html page

The snippet below is part of an HTML page. I need to scrape the data but not sure what would be the most reliable way. The best way would be JSON, but I'm not sure if the following can be converted to JSON. Is Regular Expression my only choice?
<script type="text/javascript">
window.arMailRuMessages = [];
arMailRuMessages = (function() {
var k = 1024,
u = ajs.Html.unescape,
m = function(data) {
try {
return u(decodeURIComponent(data.text));
} catch (e) {}
return '';
};
return [
{
id: "14412430340000000392",
prev: "",
next: "14412428590000000596",
subject: u("hi"),
date: "1441243034",
size: "3" | 0,
folder: "0",
correspondents: {
from: [{
name: u("firstname lastname"),
email: u("firstname#gmail.com"),
avatars: {
"default": u("\/\/filin.mail.ru\/pic?email=firstname%40gmail.com&trust=true&user=firstname%40mail.ru&sign=CA0D4E8E74E806A459EA9C793CE8BC665EB2D049")
}
}],
to: [{
name: u(""),
email: u("firstname6000#mail.ru"),
avatars: {
"default": u("")
}
}],
cc: []
},
flags: {
spf: true,
unread: true,
flagged: false,
reply: false,
forward: false,
attach: false
},
snippet: m({
"ntype": "letter",
"text": "thisisaford"
}),
priority: 3
}, {
id: "14412428590000000596",
prev: "14412430340000000392",
next: "",
subject: u("hi"),
date: "1441242859",
size: "3" | 0,
folder: "0",
correspondents: {
from: [{
name: u("firstname lastname"),
email: u("firstname#gmail.com"),
avatars: {
"default": u("\/\/filin.mail.ru\/pic?email=firstname%40gmail.com&trust=true&user=firstname%40mail.ru&sign=CA0D4E8E74E806A459EA9C793CE8BC665EB2D049")
}
}],
to: [{
name: u(""),
email: u("firstname#mail.ru"),
avatars: {
"default": u("")
}
}],
cc: []
},
flags: {
spf: true,
unread: true,
flagged: false,
reply: false,
forward: false,
attach: false
},
snippet: m({
"ntype": "letter",
"text": "thisisatest"
}),
priority: 3
}
];
})();
__log.letters_data_js = 1;
</script>

With HtmlUnit, you can use htmlPage.executeJavaScript, which will return an Object to manipulate.
Below is a complete example:
try (final WebClient webClient = new WebClient(BrowserVersion.CHROME)) {
String url = "http://localhost/test.html";
HtmlPage htmlPage = webClient.getPage(url);
NativeArray array = (NativeArray) htmlPage.executeJavaScript("arMailRuMessages").getJavaScriptResult();
for (int i = 0; i < array.getLength(); i++) {
NativeObject object = (NativeObject) array.get(i);
String id = (String) object.get("id");
System.out.println(id);
NativeObject correspondents = (NativeObject) object.get("correspondents");
NativeArray from = (NativeArray) correspondents.get("from");
System.out.println(((NativeObject) from.get(0)).get("name"));
}
}