Let's say there's some text inside a particular element on a web page that I want to save. Using Javascript, how could I save that text/append it to a file "myfile.txt" on my hard drive? The element dynamically changes over time so whenever it updates i'd like it to append the new text to the file.
I've been doing some research on web scraping, and it just seems too over the top/complicated for this task.
I've written a Node.js program that fetches a webpage url every X seconds, and compare the previous and new value of a specific html element. It will only save changes to a specific output file.
Note that the previous value record will be deleted after each run of this program, meaning that the first time you run this program it will always save the extracted text ( Because there's nothing to compare to )
This program uses node-fetch and jsdom npm packages.
fs is a build in package for Node.js.
If you are new to Node.js, you can follow this to install in your computer.
const fetch = require('node-fetch');
const jsdom = require('jsdom');
const fs = require('fs');
// Local previous extracted text variable to compare and determine changes
let prevExtractedText;
// The webpage URL to fetch from
const url = 'https://en.wikipedia.org/wiki/Node.js';
// Setting your file's output path
const outputFilepath = 'myfile.txt';
// Setting timeout every 5 seconds
const timeout = 5000;
const handleOnError = (err) => {
console.error(`! An error occurred: ${err.message}`);
process.exit(1);
}
const handleFetchAndSaveFile = async () => {
let html;
try {
console.log(`* Fetching ${url}...`);
const resp = await fetch(url);
console.log('* Converting response into html text...');
html = await resp.text();
} catch (err) {
handleOnError(err);
}
// Convert into DOM in Node.js enviroment
const dom = new jsdom.JSDOM(html);
// Example with element of id "footer-places-privacy"
const extractedText = dom.window.document.getElementById("footer-places-privacy").textContent;
console.log(`* Comparing previous extracted text (${prevExtractedText}) and current extracted text (${extractedText})`);
if (prevExtractedText !== extractedText) {
// Update prevExtractedText
prevExtractedText = extractedText;
console.log(`* Updating ${outputFilepath}...`);
try {
// Writing new extracted text into a file
await fs.appendFileSync(outputFilepath, extractedText);
console.log(`* ${outputFilepath} has been updated and saved.`);
} catch (err) {
handleOnError(err);
}
}
console.log('--------------------------------------------------')
}
console.log(`* Polling ${url} every ${timeout}ms`);
setInterval(handleFetchAndSaveFile, timeout);
Working demo: https://codesandbox.io/s/nodejs-webpage-polling-jqf6v?fontsize=14&hidenavigation=1&theme=dark
Related
I am trying to use filesystem api to create permanent files, write and read data from them.
Although I succeeded creating the file and writing data to it, after calling flush() the file becomes empty (and it's size is 0).
The files that I created exist and I can still see them in a different running of safari, but the data is lost and the file's are all 0 sized.
Even if I try to read the file just after writing to it and flushing, the data is lost and it's size returns to 0.
Does anybody know what I am doing wrong?
I tried running this:
console.log("Starting");
async function files() {
// Set a Message
const message = "Thank you for reading this.";
const fileName = "Draft.txt";
// Get handle to draft file
const root = await navigator.storage.getDirectory();
const draftHandle = await root.getFileHandle(fileName, { create: true });
console.log("File Name: ", fileName);
// Get sync access handle
const accessHandle = await draftHandle.createSyncAccessHandle();
// Get size of the file.
const fileSize = await accessHandle.getSize();
console.log("File Size: ", fileSize); // getting 0 here
// Read file content to a buffer.
const buffer = new DataView(new ArrayBuffer(fileSize));
const readBuffer = accessHandle.read(buffer, { at: 0 });
console.log("Read: ", readBuffer); // getting 0 here because the file was just created
// Write the message to the file.
const encoder = new TextEncoder();
const encodedMessage = encoder.encode(message);
const writeBuffer = accessHandle.write(encodedMessage, { at: readBuffer });
console.log("Write: ", writeBuffer); // writing 27 bytes here, succeeding
// Persist changes to disk.
accessHandle.flush();
// Always close FileSystemSyncAccessHandle if done.
accessHandle.close();
console.log("Closed file ");
// Find files under root/ and print their details.
for await (const handle of root.values()) {
console.log('Item in root: ', handle.name);
if (handle.kind == "file") {
let f = await handle.getFile();
console.log("File Details: ", f); // I can see here the file I created now and earlier created files, but all of them are 0 sized
}
}
}
files();
I'm using the azure file storage, and using express JS to write a backend to render the contents stored in the azure file storage.
I am writing the code based on https://learn.microsoft.com/en-us/javascript/api/#azure/storage-file-share/shareserviceclient?view=azure-node-latest
const { ShareServiceClient, StorageSharedKeyCredential } = require("#azure/storage-file-share");
const account = "<account>";
const accountKey = "<accountkey>";
const credential = new StorageSharedKeyCredential(account, accountKey);
const serviceClient = new ShareServiceClient(
`https://${account}.file.core.windows.net`,
credential
);
const shareName = "<share name>";
const fileName = "<file name>";
// [Node.js only] A helper method used to read a Node.js readable stream into a Buffer
async function streamToBuffer(readableStream) {
return new Promise((resolve, reject) => {
const chunks = [];
readableStream.on("data", (data) => {
chunks.push(data instanceof Buffer ? data : Buffer.from(data));
});
readableStream.on("end", () => {
resolve(Buffer.concat(chunks));
});
readableStream.on("error", reject);
});
}
And you can view the contents through
const downloadFileResponse = await fileClient.download();
const output = await streamToBuffer(downloadFileResponse.readableStreamBody)).toString()
Thing is, I only want to find if the file exists and not spend time downloading the entire file, how could I do this?
I looked at https://learn.microsoft.com/en-us/javascript/api/#azure/storage-file-share/shareserviceclient?view=azure-node-latest
to see if the file client class has what I want, but it doesn't seem to have methods useful for this.
If you are using #azure/storage-file-share (version 12.x) Node package, there's an exists method in ShareFileClient. You can use that to find if a file exists or not. Something like:
const fileExists = await fileClient.exists();//returns true or false.
I'm currently getting to grips with the File System Access API for a future project, and I am having difficulty using the createWritable() method in the line:
const writable = await fileHandle.createWritable();
When I trigger it, it gives me the error:
TypeError: fileHandle.createWritable is not a function
However, when I run this exact command in the console, it works perfectly.
I'm using windows 10 if that's relevant at all. Thanks for the help.
I will include my relevant HTML and JS below:
HTML:
<body>
<h1>Hello There</h1>
<button class="read-btn">Read</button>
<textarea id="txt"></textarea>
<button class="create-btn">Create</button>
<button class="write-btn">Write</button>
</body>
<script src="./index.js"></script>
JS:
const readBtn = document.querySelector(".read-btn");
const createBtn = document.querySelector(".create-btn")
const writeBtn = document.querySelector(".write-btn");
const txt = document.querySelector("#txt");
// Store a ref to file handle
let fileHandle;
let contents;
// === READ FROM A FILE SYSTEM ===
readBtn.addEventListener("click", async () => {
// open filepicker
[fileHandle] = await window.showOpenFilePicker();
// Returns a file object
const file = await fileHandle.getFile();
// Runs text method on returned file object to access text
contents = await file.text();
// Inputs contents into txt
txt.value = contents;
});
// === WRITE TO FILESYSTEM ===
// Create a new file
async function getNewFileHandle() {
const options = {
types: [
{
description: 'Text Files',
accept: {
'text/plain': ['.txt'],
},
},
],
};
const handle = await window.showSaveFilePicker(options);
return handle;
}
// Save changes to disk
async function writeFile(fileHandle) {
contents = txt.value;
// Create a FileSystemWritableFileStream to write to.
const writable = await fileHandle.createWritable();
// Write the contents of the file to the stream.
await writable.write(contents);
// Close the file and write the contents to disk.
await writable.close();
}
createBtn.addEventListener("click", getNewFileHandle)
writeBtn.addEventListener("click", writeFile)
According to MDN documentation, window.showOpenFilePicker, FileSystemFileHandle, and FileSystemWritableFileStream are only supported in secure contexts:
Secure context
This feature is available only in secure contexts
(HTTPS), in some or all supporting
browsers.
It's my understanding that the "Live Server" extension for VS Code would not meet this requirement, but the developer console would.
How to give an argument to command explorer.newFile in Visual Studio Code extension development?
I just want to add create a file with certain name in explorer view and then open it and insert a snippet, these operations will be realized in a function .Help ^_^
I could certainly be wrong but I am not sure explorer.newFile will take an argument.
Nevertheless, this function will do what you want: create a file, open it, and insert an existing snippet:
async function createFileOpen() {
const we = new vscode.WorkspaceEdit();
const thisWorkspace = await vscode.workspace.workspaceFolders[0].uri.toString();
// if you want it to be in some folder under the workspaceFolder: append a folder name
// const uriBase = `${thisWorkspace}/folderName`;
// let newUri1 = vscode.Uri.parse(`${uriBase}/index.js`);
// create a Uri for a file to be created
const newUri = await vscode.Uri.parse(`${ thisWorkspace }\\myTestIndex.js`);
// create an edit that will create a file
await we.createFile(newUri, { ignoreIfExists: false, overwrite: true });
await vscode.workspace.applyEdit(we); // actually apply the edit: in this case file creation
await vscode.workspace.openTextDocument(newUri).then(
async document => {
await vscode.window.showTextDocument(document);
// if you are using a predefined snippet
await vscode.commands.executeCommand('editor.action.insertSnippet', { 'name': 'My Custom Snippet Label Here'});
});
}
I'm using electron to develop an app. after some encryption operations are done, I need to show a dialog to the user to save the file. The filename I want to give to the file is a random hash but I have no success also with this. I'm trying with this code but the file will not be saved. How I can fix this?
const downloadPath = app.getPath('downloads')
ipcMain.on('encryptFiles', (event, data) => {
let output = [];
const password = data.password;
data.files.forEach( (file) => {
const buffer = fs.readFileSync(file.path);
const dataURI = dauria.getBase64DataURI(buffer, file.type);
const encrypted = CryptoJS.AES.encrypt(dataURI, password).toString();
output.push(encrypted);
})
const filename = hash.createHash('md5').toString('hex');
console.log(filename)
const response = output.join(' :: ');
dialog.showSaveDialog({title: 'Save encrypted file', defaultPath: downloadPath }, () => {
fs.writeFile(`${filename}.mfs`, response, (err) => console.log(err) )
})
})
The problem you're experiencing is resulting from the asynchronous nature of Electron's UI functions: They do not take callback functions, but return promises instead. Thus, you do not have to pass in a callback function, but rather handle the promise's resolution. Note that this only applies to Electron >= version 6. If you however run an older version of Electron, your code would be correct -- but then you should really update to a newer version (Electron v6 was released well over a year ago).
Adapting your code like below can be a starting point to solve your problem. However, since you do not state how you generate the hash (where does hash.createHash come from?; did you forget to declare/import hash?; did you forget to pass any message string?; are you using hash as an alias for NodeJS' crypto module?), it is (at this time) impossible to debug why you do not get any output from console.log (filename) (I assume you mean this by "in the code, the random filename will not be created"). Once you provide more details on this problem, I'd be happy to update this answer accordingly.
As for the default filename: As per the Electron documentation, you can pass a file path into dialog.showSaveDialog () to provide the user with a default filename.
The file type extension you're using should also actually be passed with the file extension into the save dialog. Also passing this file extension as a filter into the dialog will prevent users from selecting any other file type, which is ultimately what you're also currently doing by appending it to the filename.
Also, you could utilise CryptoJS for the filename generation: Given some arbitrary string, which could really be random bytes, you could do: filename = CryptoJS.MD5 ('some text here') + '.mfs'; However, remember to choose the input string wisely. MD5 has been broken and should thus no longer be used to store secrets -- using any known information which is crucial for the encryption of the files you're storing (such as data.password) is inherently insecure. There are some good examples on how to create random strings in JavaScript around the internet, along with this answer here on SO.
Taking all these issues into account, one might end up with the following code:
const downloadPath = app.getPath('downloads'),
path = require('path');
ipcMain.on('encryptFiles', (event, data) => {
let output = [];
const password = data.password;
data.files.forEach((file) => {
const buffer = fs.readFileSync(file.path);
const dataURI = dauria.getBase64DataURI(buffer, file.type);
const encrypted = CryptoJS.AES.encrypt(dataURI, password).toString();
output.push(encrypted);
})
// not working:
// const filename = hash.createHash('md5').toString('hex') + '.mfs';
// alternative requiring more research on your end
const filename = CryptoJS.MD5('replace me with some random bytes') + '.mfs';
console.log(filename);
const response = output.join(' :: ');
dialog.showSaveDialog(
{
title: 'Save encrypted file',
defaultPath: path.format ({ dir: downloadPath, base: filename }), // construct a proper path
filters: [{ name: 'Encrypted File (*.mfs)', extensions: ['mfs'] }] // filter the possible files
}
).then ((result) => {
if (result.canceled) return; // discard the result altogether; user has clicked "cancel"
else {
var filePath = result.filePath;
if (!filePath.endsWith('.mfs')) {
// This is an additional safety check which should not actually trigger.
// However, generally appending a file extension to a filename is not a
// good idea, as they would be (possibly) doubled without this check.
filePath += '.mfs';
}
fs.writeFile(filePath, response, (err) => console.log(err) )
}
}).catch ((err) => {
console.log (err);
});
})