I am trying to stream a text file processed in NodeJS to a browser.
The following is the text file before processing.
The file is named dbUsers.json.
{"userId":443,"email":"bob#gmail.com","hashedPassword":"36583a77a098c02ef111e2f2521d77b58e420f2bc7e9bf930ec24b21d42ea2e0","timeStamp":1567439821109,"deleted":false}
{"userId":447,"email":"alice#gmail.com","hashedPassword":"36583a77a098c02ef111e2f2521d77b58e420f2bc7e9bf930ec24b21d42ea2e0","timeStamp":1567439909013,"deleted":false}
{"userId":451,"email":"cliff#gmail.com","hashedPassword":"36583a77a098c02ef111e2f2521d77b58e420f2bc7e9bf930ec24b21d42ea2e0","timeStamp":1567443638340,"deleted":false}
...
After processing I am able to stream the data to a new file on the NodeJS server with the following commands:
// Create a writable stream and specify the file which will receive the data from the readable stream.
let destinationStream = fs.createWriteStream(_data.baseDir + '/dbPermissions/dbUsers' + '/' + 'test' + '.txt', {flags : 'a'});
pipeline
(
sourceStream,
destinationStream,
function(error){if(error){console.log('There was an error.');}}
);
The new file shows the data processed as expected.
Some fields have been deleted and the records marked for deletion have been removed.
This demonstrates that sourceStream is functioning within NodeJS.
The data in the new file is now as follows:
{"userId":443,"email":"bob#gmail.com","timeStamp":1567439821109}
{"userId":447,"email":"alice#gmail.com","timeStamp":1567439909013}
{"userId":451,"email":"cliff#gmail.com","timeStamp":1567443638340}
...
Logging sourceStream to the NodeJS console before streaming it to the client browser produces the following output.
Readable {
_readableState:
ReadableState {
objectMode: false,
highWaterMark: 16384,
buffer: BufferList { head: [Object], tail: [Object], length: 45 },
length: 3035,
pipes:
WriteStream {
_writableState: [WritableState],
writable: true,
domain: null,
_events: [Object],
_eventsCount: 6,
_maxListeners: undefined,
path:
'C:\\Users\\user\\Desktop\\Tutorials\\iotajs\\ias\\accounting\\/dbPermissions/dbUsers/test.txt',
fd: null,
flags: 'a',
mode: 438,
start: undefined,
autoClose: true,
pos: undefined,
bytesWritten: 0,
closed: false },
pipesCount: 1,
flowing: true,
ended: true,
endEmitted: false,
reading: false,
sync: true,
needReadable: false,
emittedReadable: true,
readableListening: false,
resumeScheduled: true,
paused: false,
emitClose: true,
destroyed: false,
defaultEncoding: 'utf8',
awaitDrain: 0,
readingMore: true,
decoder: null,
encoding: null },
readable: true,
domain: null,
_events:
[Object: null prototype] {
close: [ [Function], [Function: onclose] ],
end: [ [Function: onend], [Function] ],
finish: [Function: onfinish],
error: [Function: onerror],
data: [Function: ondata] },
_eventsCount: 5,
_maxListeners: undefined }
Returning this response: 200
Returning this response: 200
When streaming sourceStream to the browser and then logging out to the browser's console, the output is the same as above.
So I am sure that sourceStream is getting to the client intact.
The data I need to work with is probably locked up in the many
buffer properties of the object above, now called responseTextStream on
the client browser.
My problem is that I don't know how to access the information in the
buffers nor do I know how to convert them from numbers back into text.
The following is the function in the client browser where I am hoping
to use the data in the buffers. This is where I need help - I don't know how to access the stream. Thanks, John
// Populate the dbUsersList webpage with user records.
app.loadUsersListPage = function()
{
// Ask the server for the JSON records found in the dbUsers file.
// Then run the callback function defined here which inserts rows into the usersListTable on the webpage
// and populates them with data from the file of JSON records returned.
app.client.request(undefined,'api/aUsers','GET',QueryStringObject,undefined,function(statusCode,responseTextStream)
{
// if the call to handlers._users.get which is mapped to api/aUsers called back success.
if(statusCode == 200)
{
// The streamed data can be seen on the console as a buffer full of numbers
console.log(responseTextStream._readableState.buffer.head.data.data);
// Create a handle which can be used to manipulate the table on the webpage.
var table = document.getElementById("usersListTable");
// The pseudocode below does not work but is shows what I hope to accomplish.
// The line below does not help to access the stream. This is where I need help.
// What line or lines of code would facilitate access to the stream and allow
// processing it as a string, character by character, as shown below.
var Astr = responseTextStream;
var line = "";
for(var i=0; i<Astr.length; i++)
{
var chr = String.fromCharCode(Astr[i]);
if(chr == "\n" || chr == "\r")
{
// Look at each line of json at the console as it is consumed.
console.log("line: ",line);
// Turn the line, which is a json string, back into a json object
var recordObject = JSON.parse(line);
if(recordObject)
{
// Insert a new row in the table.
var tr = table.insertRow(-1);
// Make the new row a member of the class 'checkRow'
tr.classList.add('checkRow');
// Insert five new cells into the new row.
var td0 = tr.insertCell(0);
var td1 = tr.insertCell(1);
var td2 = tr.insertCell(2);
var td3 = tr.insertCell(3);
// load the new cells with data from the recordObject.
td0.innerHTML = recordObject.userId;
td1.innerHTML = recordObject.email;
td2.innerHTML = recordObject.timeStamp;
td3.innerHTML = 'View / Edit / Delete';
} // End of: if(recordObject)
// clear the line buffer to start the next line.
line = "";
} // End of: if(chr == "\n" || chr == "\r"){do stuff}
else
{
line += chr;
}
}; // End of: for(var i=0; i<Astr.length; i++){...}
} // End of: if the call to handlers._users.get which is mapped to api/aUsers called back successfully.
}); // End of: app.client.request(undefined,'api/checks','GET'...
} // End of: app.loadUsersListPage = function(){...}
// End of: Populate the dbUsersList webpage with user records.
According to the answer from #Brad I used his code with the same
unsatisfactory result. Below is Brad's code and below that is the
object that is returned by his code at the point where value is
logged to the console. It's the same object as before with the buffers
expanded for examination. My problem remains that the returned object
is just a bunch of buffers that contain a bunch of numbers and not
strings that I can do something with. When logging value to the
console I am hoping to see each line of the modified data in human
readable format. What am I not doing, or what am I doing wrong?
Thanks, John
// Populate the dbUsersList webpage with user records.
app.loadUsersListPage = async function()
{
// Define which users will be retrieved from dbUsers.json
// This is not being used for now so all records will be retrived.
var QueryStringObject = {};
// Define a client function that calls for data from the server.
const fetchPromise = fetch('api/aUsers')
.then
(
(res) =>
{
// Verify that we have some sort of 2xx response that we can use
if (!res.ok)
{
throw res;
}
// If no content, immediately resolve, don't try to parse JSON
if (res.status === 204)
{
return;
}
// Initialize variable to hold chunks of data as they come across.
let textBuffer = '';
// This does not seem to be used. Delete this after everything else is working.
const self = this;
// Process the stream.
return res.body
// Decode as UTF-8 Text
.pipeThrough
(
new TextDecoderStream()
)
// Split on lines
.pipeThrough
(
new TransformStream
(
{
transform(chunk, controller)
{
textBuffer += chunk;
const lines = textBuffer.split('\n');
for (const line of lines.slice(0, -1))
{
controller.enqueue(line);
} // End of: for (const line ...)
textBuffer = lines.slice(-1)[0];
}, // End of: Transform(chunk, controller){do stuff}
flush(controller)
{
if (textBuffer)
{
controller.enqueue(textBuffer);
} // End of: if (textBuffer)
} // End of: flush(controller){do stuff}
} // End of: parameters for new TransformStream
) // End of: call to constructor new TransformStream
) // End of: parameters for pipeThrough - Split on lines
// Parse JSON objects
.pipeThrough
(
new TransformStream
(
{
transform(line, controller)
{
if (line)
{
controller.enqueue
(
JSON.parse(line)
); //End of: call to controller.enqueue function
} // End of: if (line)
} // End of: transform function
} // End of: parameter object for new TransformStream
) // End of: new TransformStream parameters
); // End of: parameters for .pipeThrough - Parse JSON objects
} // End of: .then callback function instruction for fetch
); // End of: .then callback parameters for fetch
// Call to function which asks server for data.
const res = await fetchPromise;
const reader = res.getReader();
function read()
{
reader.read()
.then
(
({value, done}) =>
{
if (value) {
// Your object will be here
console.log('I got to this point');
console.log(value);
}
if (done) {
return;
}
read();
}
);
}
read();
} // End of: app.loadUsersListPage = function(){...}
// End of: Populate the dbUsersList webpage with user records.
This is what I get when Brad's code logs value to the console. It is
the same thing I got before. I was hoping to see lines of text. What
am I not doing, or what am I doing wrong? Thanks, John
{_readableState: {…}, readable: true, domain: null, _events: {…}, _eventsCount: 0}
domain: null
readable: true
_events: {}
_eventsCount: 0
_readableState:
awaitDrain: 0
buffer:
head:
data:
data: (65) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 52, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 111, 98, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 51, 57, 56, 50, 49, 49, 48, 57, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 52, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 97, 108, 105, 99, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 51, 57, 57, 48, 57, 48, 49, 51, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 53, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 99, 108, 105, 102, 102, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 51, 54, 51, 56, 51, 52, 48, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 53, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 114, 103, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 51, 55, 54, 53, 54, 48, 57, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 53, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 99, 108, 105, 110, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 51, 56, 49, 51, 49, 54, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 54, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 121, 114, 111, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 51, 57, 52, 50, 48, 57, 54, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 54, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 105, 108, 116, 111, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 52, 50, 48, 55, 53, 52, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 55, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 97, 114, 110, 111, 108, 100, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 52, 52, 50, 51, 57, 55, 53, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 55, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 115, 97, 108, 108, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 52, 56, 57, 56, 57, 52, 54, 57, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 57, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 114, 118, 105, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 53, 48, 51, 56, 50, 52, 54, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 57, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 110, 97, 110, 99, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 53, 48, 52, 49, 55, 52, 52, 57, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 52, 57, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 102, 114, 101, 100, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 53, 48, 52, 51, 48, 55, 52, 50, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 48, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 97, 108, 101, 120, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 52, 53, 52, 56, 55, 57, 55, 53, 51, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 48, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 104, 101, 114, 109, 97, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 53, 48, 54, 48, 50, 55, 50, 49, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 49, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 114, 103, 114, 101, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 53, 53, 55, 53, 50, 49, 51, 55, 51, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 49, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 115, 121, 110, 116, 104, 105, 97, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 53, 53, 55, 53, 55, 51, 56, 52, 48, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 49, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 99, 111, 110, 110, 101, 114, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 53, 53, 55, 54, 48, 55, 51, 56, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 50, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 114, 103, 111, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 55, 53, 53, 56, 53, 57, 57, 50, 48, 48, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 50, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 100, 101, 110, 105, 115, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 48, 54, 54, 50, 56, 55, 53, 54, 48, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 51, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 112, 104, 105, 108, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 48, 54, 54, 56, 49, 50, 48, 55, 48, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 51, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 111, 114, 103, 97, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 48, 55, 55, 53, 55, 55, 56, 53, 56, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 51, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 108, 117, 107, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 48, 55, 50, 53, 48, 49, 51, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 52, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 119, 97, 114, 114, 101, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 48, 56, 52, 55, 50, 54, 56, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 53, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 119, 105, 108, 98, 117, 114, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 49, 49, 49, 56, 54, 51, 55, 54, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 53, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 112, 97, 117, 108, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 49, 49, 50, 50, 52, 55, 57, 49, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (70) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 53, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 102, 108, 111, 114, 97, 110, 99, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 53, 50, 50, 48, 48, 51, 50, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 54, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 117, 109, 102, 111, 114, 100, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 53, 50, 51, 55, 53, 57, 56, 50, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 54, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 97, 114, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 53, 55, 55, 48, 49, 57, 55, 49, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 55, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 104, 111, 109, 101, 114, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 53, 55, 55, 57, 48, 53, 49, 53, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 55, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 102, 97, 108, 107, 110, 101, 114, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 54, 51, 48, 52, 55, 52, 53, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (70) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 55, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 97, 110, 100, 101, 114, 115, 111, 110, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 57, 54, 51, 56, 53, 50, 53, 51, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 56, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 97, 114, 114, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 49, 57, 55, 55, 52, 54, 48, 55, 51, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 56, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 97, 114, 110, 101, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 50, 51, 54, 54, 52, 54, 54, 54, 50, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 57, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 101, 116, 116, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 50, 51, 54, 55, 52, 57, 54, 51, 54, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 57, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 115, 116, 101, 118, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 50, 52, 52, 56, 49, 56, 48, 50, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 53, 57, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 115, 104, 105, 114, 108, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 50, 56, 49, 54, 49, 54, 50, 53, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 48, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 103, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 50, 56, 49, 56, 57, 54, 52, 50, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (71) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 48, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 99, 111, 110, 115, 116, 97, 110, 99, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 50, 48, 54, 56, 49, 53, 54, 54, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (68) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 49, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 109, 97, 114, 115, 104, 97, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 50, 48, 56, 55, 55, 57, 56, 54, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 49, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 103, 114, 101, 103, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 50, 49, 49, 53, 55, 49, 56, 53, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 50, 51, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 112, 101, 116, 101, 114, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 50, 49, 50, 48, 55, 55, 51, 55, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (67) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 50, 55, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 99, 105, 110, 100, 121, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 56, 52, 52, 52, 49, 52, 54, 49, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 51, 49, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 114, 111, 115, 101, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 56, 52, 52, 55, 50, 49, 48, 57, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (66) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 51, 53, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 98, 97, 114, 98, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 56, 52, 53, 51, 57, 52, 50, 52, 125, 10]
type: "Buffer"
__proto__: Object
next:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 51, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 103, 105, 108, 98, 101, 114, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 56, 52, 53, 57, 56, 56, 54, 55, 125, 10]
type: "Buffer"
__proto__: Object
next: null
length: 45
tail:
data:
data: (69) [123, 34, 117, 115, 101, 114, 73, 100, 34, 58, 54, 51, 57, 44, 34, 101, 109, 97, 105, 108, 34, 58, 34, 103, 105, 108, 98, 101, 114, 116, 64, 103, 109, 97, 105, 108, 46, 99, 111, 109, 34, 44, 34, 116, 105, 109, 101, 83, 116, 97, 109, 112, 34, 58, 49, 53, 54, 56, 52, 56, 52, 53, 57, 56, 56, 54, 55, 125, 10]
type: "Buffer"
__proto__: Object
next: null
I did not include the entire object.
As seen above, many buffers are contained in the object returned. I
think each buffer contains a line/record from the original dbUsers
file with some fields removed and duplicate records deleted. I would
like to see these buffers as human readable text. Because many
buffers are being returned, eventually I will need to learn how to
iterate through the buffers but for now I am just trying turn the
first buffer into text. Brad at this point is at a disadvantage
because he is not near a computer. So he can not see the code I am
using. In any case, he kindly advised me to try the toString()
function. So I tried the following line of code on the first buffer.
console.log(value._readableState.buffer.head.data.data.toString());
The following numbers were the result. I was hoping for a string of
text. Can anyone see what I am failing to do? Thanks, John
123,34,117,115,101,114,73,100,34,58,52,52,51,44,34,101,109,97,105,108,34,58,34,98,111,98,64,103,109,97,105,108,46,99,111,109,34,44,34,116,105,109,101,83,116,97,109,112,34,58,49,53,54,55,52,51,57,56,50,49,49,48,57,125,10
After trying the following line of code I am still getting numbers not
letters at the console.
console.log(value._readableState.buffer.head.data.data.toString('utf8'));
Thanks, John
Since you're using Chrome, you get to use all the new goodness like TextDecoderStream and TransformStream in a nifty pipeline that streams data from the HTTP response and decodes that line-delimited JSON as it does it. Check this out:
const fetchPromise = fetch(url, params).then((res) => {
// Verify that we have some sort of 2xx response that we can use
if (!res.ok) {
throw res;
}
// If no content, immediately resolve, don't try to parse JSON
if (res.status === 204) {
return;
}
let textBuffer = '';
const self = this;
return res.body
// Decode as UTF-8 Text
.pipeThrough(new TextDecoderStream())
// Split on lines
.pipeThrough(new TransformStream({
transform(chunk, controller) {
textBuffer += chunk;
const lines = textBuffer.split('\n');
for (const line of lines.slice(0, -1)) {
controller.enqueue(line);
}
textBuffer = lines.slice(-1)[0];
},
flush(controller) {
if (textBuffer) {
controller.enqueue(textBuffer);
}
}
}))
// Parse JSON objects
.pipeThrough(new TransformStream({
transform(line, controller) {
if (line) {
controller.enqueue(
JSON.parse(line)
);
}
}
}));
});
Now, you can consume this new object stream like any other:
const res = await fetchPromise;
const reader = res.getReader();
function read() {
reader.read().then(({value, done}) => {
if (value) {
// Your object will be here
}
if (done) {
return;
}
read();
});
}
read();
(Note: I haven't tested this code in this example context... I modified this from one of my projects, so please look it over and adapt it for your specific purposes.)
Brad's JavaScript from above which runs in the browser worked perfectly.
The problem was the way in which the stream had been sent from the NodeJS server to the browser.
All the working code has been pushed to github.
Click here to see the full repository which uses no dependencies.
The action starts here with the function called loadUsersListPage
The function is listed below for convenience.
This is Brad's code which requests a stream of text from the server and then consumes the stream to populate a webpage with a list of users from the dbUsers table on the server.
// Populate the dbUsersList webpage with user records.
app.loadUsersListPage = async function()
{
// Create a handle which can be used to manipulate the table on the webpage.
var table = document.getElementById("usersListTable");
// Define which users will be retrieved from dbUsers.json
// This is not being used for now so all records will be retrived.
var QueryStringObject = {};
// Define a client function that calls for data from the server.
const fetchPromise = fetch('api/aUsers')
.then
(
(res) =>
{
// Verify that we have some sort of 2xx response that we can use
if (!res.ok)
{
// throw res;
// Show 'you have no checks' message
document.getElementById("noChecksMessage").style.display = 'table-row';
// Show the createCheck CTA
document.getElementById("createCheckCTA").style.display = 'block';
console.log("Error trying to load the list of users: ");
}
// If no content, immediately resolve, don't try to parse JSON
if (res.status === 204)
{
return;
}
// Initialize variable to hold chunks of data as they come across.
let textBuffer = '';
// Process the stream.
return res.body
// Decode as UTF-8 Text
.pipeThrough
(
new TextDecoderStream()
)
// Split on lines
.pipeThrough
(
new TransformStream
(
{
transform(chunk, controller)
{
textBuffer += chunk;
// Split the string of records on the new line character and store the result in an array named lines.
const lines = textBuffer.split('\n');
// Cycle through all elements in the array except for the last one which is only holding a new line character.
for (const line of lines.slice(0, -1))
{
// Put the element from the array into the controller que.
controller.enqueue(line);
} // End of: for (const line ...)
// Put the last element from the array (the new line character) into the textBuffer but don't put it in the que.
textBuffer = lines.slice(-1)[0];
}, // End of: Transform(chunk, controller){do stuff}
flush(controller)
{
if (textBuffer)
{
controller.enqueue(textBuffer);
} // End of: if (textBuffer)
} // End of: flush(controller){do stuff}
} // End of: parameters for new TransformStream
) // End of: call to constructor new TransformStream
) // End of: parameters for pipeThrough - Split on lines
// Parse JSON objects
.pipeThrough
(
new TransformStream
(
{
transform(line, controller)
{
if (line)
{
controller.enqueue
(
JSON.parse(line)
); //End of: call to controller.enqueue function
} // End of: if (line)
} // End of: transform function
} // End of: parameter object for new TransformStream
) // End of: new TransformStream parameters
); // End of: parameters for .pipeThrough - Parse JSON objects
} // End of: .then callback function instruction for fetch
); // End of: .then callback parameters for fetch
// Call to function which asks server for data.
const res = await fetchPromise;
const reader = res.getReader();
function read()
{
reader.read()
.then
(
({value, done}) =>
{
if (value)
{
// Your object (value) will be here
// Insert a new row in the table.
var tr = table.insertRow(-1);
// Make the new row a member of the class 'checkRow'
tr.classList.add('checkRow');
// Insert five new cells into the new row.
var td0 = tr.insertCell(0);
var td1 = tr.insertCell(1);
var td2 = tr.insertCell(2);
var td3 = tr.insertCell(3);
// load the new cells with data from the recordObject.
td0.innerHTML = value.userId;
td1.innerHTML = value.email;
td2.innerHTML = value.timeStamp;
td3.innerHTML = 'View / Edit / Delete';
} // End of: if(value){do stuff}
if (done) {return;}
read();
// Show the createCheck CTA
document.getElementById("createCheckCTA").style.display = 'block';
} // End of: if a record object (value) is returned.
); // End of: .then callback after read function completes.
} // End of: function definition: function read(){do stuff}
// Call the read function defined above.
read();
} // End of: app.loadUsersListPage = function(){...}
// End of: Populate the dbUsersList webpage with user records.
The working server code is found here under the function named unifiedServer
The function is listed directly below for convenience.
This function routes the request to a handler (also listed below) and then sends the
stream of returned data back to the browser for Brad's code to
consume.
Near the bottom of the function you can see a pipeline is used to send the stream to the browser.
This is what I was missing.
// Define a function to route requests from the client to the handler and to serve back a response.
// All the logic for both the http and https server
server.unifiedServer = function(req, res)
{
// Get the URL and parse it.
var parsedUrl = url.parse(req.url, true);
// Get the path from the URL.
var path = parsedUrl.pathname;
var trimmedPath = path.replace(/^\/+|\/+$/g, '');
// Get the query string as an object.
var queryStringObject = parsedUrl.query;
// Get the http method.
var method = req.method.toLowerCase();
// Get the headers as an Object
var headers = req.headers;
// Instantiate decoder that will turn payload buffer into a string.
var decoder = new StringDecoder('utf8');
// Create an empty string for the request payload.
// It's called buffer but it is not a JavaScript buffer (not binary data) - it's just an empty string.
// We are going to use it to hold the request payload buffer after it has been decoded and turned into a string.
var buffer = '';
// Call to event emitter req.on('data...
// Watch for a chunk of data from the client's request payload buffer.
// The run the callback defined below with the returned data.
req.on('data', function(data)
{
// Decode the chunk and write it to the payload string.
buffer += decoder.write(data);
});
// Call to event emitter req.on('end...
// Watch for the end of the payload from the client request.
// The callback defined here is the action taken after the entire request has been received.
req.on('end', function()
{
// Finish writing to the buffer.
buffer += decoder.end();
// Choose the handler the client's request should go to.
// If one is not found, use the notFound handler.
// To be clear: A key in the router object below should match the request from the client.
// chosenHandler becomes an alias for the handler function which is mapped to the key in the router object.
// This is how we can refer to the handler function without knowing what it is in advance.
var chosenHandler = typeof(server.router[trimmedPath]) !== 'undefined' ? server.router[trimmedPath] : aHandlers.notFound;
// If the request is within the public directory, use the public handler instead of what was assigned by the line above.
// The line below is required because the line above will only match a handler to a client request if
// the request exactly matches one of the keys in the request router object at the bottom of this file.
// So if the request is "public" then we have a match with the public handler. But request
// public/app.css would not be matched with a handler.
chosenHandler = trimmedPath.indexOf('public/') > -1 ? handlers.public : chosenHandler;
// Construct the data object to send to the handler.
var data =
{
'trimmedPath' : trimmedPath,
'queryStringObject' : queryStringObject,
'method' : method,
'headers' : headers,
'payload' : aHelpers.parseJsonToObject(buffer)
}; // End of: Construct the data object to send to the handler.
// Call the handler specified by the client.
// Then execute the callback we are passing in as an argument.
chosenHandler(data, function(statusCode, payload, contentType)
{
// Determine the type of response - default to json.
contentType = typeof(contentType) == 'string' ? contentType : 'json';
// Use the status code called back by the handler, or default to 200.
statusCode = typeof(statusCode) == 'number' ? statusCode : 200;
// Return the response parts that are content specific.
var payloadString = '';
if(contentType == 'json')
{
res.setHeader('Content-Type', 'application/json');
// Use the payload called back by the handler, or default to an empty object.
payload = typeof(payload) =='object' ? payload : {};
// Convert the payload to a string.
payloadString = JSON.stringify(payload);
}
if(contentType == 'html')
{
res.setHeader('Content-Type', 'text/html');
payloadString = typeof(payload) == 'string' ? payload : '';
}
if(contentType == 'favicon')
{
res.setHeader('Content-Type', 'image/x-icon');
payloadString = typeof(payload) !== 'undefined' ? payload : '';
}
if(contentType == 'css')
{
res.setHeader('Content-Type', 'text/css');
payloadString = typeof(payload) !== 'undefined' ? payload : '';
}
if(contentType == 'png')
{
res.setHeader('Content-Type', 'image/png');
payloadString = typeof(payload) !== 'undefined' ? payload : '';
}
if(contentType == 'jpg')
{
res.setHeader('Content-Type', 'image/jpeg');
payloadString = typeof(payload) !== 'undefined' ? payload : '';
}
if(contentType == 'plain')
{
res.setHeader('Content-Type', 'text/plain');
payloadString = typeof(payload) !== 'undefined' ? payload : '';
}
// Return the response parts that are common to all content-types.
res.writeHead(statusCode);
if (contentType == 'stream')
{
pipeline
(
payload,
res,
function(error){if(error){console.log('There was an error.');}}
);
}
else
{
res.end(payloadString);
}
console.log('Returning this response: ', statusCode);
}); // End of: call to chosenHandler(...
}); // End of: call to req.on('end', function(...
}; // End of: var unifiedServer = function(...
// End of: Define a function to route requests from the client to the handler and to serve back a response.
The handler function which:
reads data from the dbUsers.json file,
processes the data,
and returns it as a stream
can be found here under the name handlers._users.get
The function is listed below for convenience.
handlers._users.get = function(data, callback)
{
// Create an empty map data structure which will be used to merge user records with the same email address.
let usersMap = new Map();
// This function sets up a stream where each chunk of data is a complete line in the dbUsers file.
let readInterface = readline.createInterface
(
{ // specify the file to be read.
input: fs.createReadStream(_data.baseDir + '/dbPermissions/dbUsers' + '/' + 'dbUsers' + '.json'),
}
);
// Look at each record in the file.
readInterface.on('line', function(line)
{
// Convert the JSON string (a single line from the dbUsers file) into lineValueObject.
// These objects will written back to a new file after deleting some un-needed key/value pairs.
let lineValueObject = JSON.parse(line);
// Declare a variable to serve as a key in the map to manage the lineValueObject.
let userId = lineValueObject.userId;
if(lineValueObject.deleted === true)
{
// Remove this record from the map
usersMap.delete(userId);
}
else // this record has not been marked for deletion.
{
// Remove the hashed password key/value pair from the lineValueObject before returning it to the requester.
delete lineValueObject.hashedPassword;
// Remove the deleted key/value pair from the lineValueObject before returning it to the requester.
delete lineValueObject.deleted;
// Update this record in the map.
usersMap.set(userId, lineValueObject);
}
}); // End of: readInterface.on('line', function(line){...}
// End of: Look at each record...
// This listener fires after we have looked through all the records in the dbUsers file.
// The callback function defined here will stream the list of users back to the clients browser.
readInterface.on('close', function()
{
// This readable stream will be used to write the result of the merge to a new file.
const sourceStream = new Readable();
for (const [key, valueObject] of usersMap)
{
// Convert the data object to a string.
let stringData = JSON.stringify(valueObject);
// Load the readable stream with data.
sourceStream.push(stringData + '\n');
}
// Tell the stream no more data is coming.
sourceStream.push(null);
callback(200, sourceStream, 'stream');
}); // End of: readInterface.on('close', function(){...}
}; // End of: handlers._users.get = function(data, callback){do stuff}