Node can't find certain modules after synchronous install - javascript

I've got a script that synchronously installs non-built-in modules at startup that looks like this
const cp = require('child_process')
function requireOrInstall (module) {
try {
require.resolve(module)
} catch (e) {
console.log(`Could not resolve "${module}"\nInstalling`)
cp.execSync(`npm install ${module}`)
console.log(`"${module}" has been installed`)
}
console.log(`Requiring "${module}"`)
try {
return require(module)
} catch (e) {
console.log(require.cache)
console.log(e)
}
}
const http = require('http')
const path = require('path')
const fs = require('fs')
const ffp = requireOrInstall('find-free-port')
const express = requireOrInstall('express')
const socket = requireOrInstall('socket.io')
// List goes on...
When I uninstall modules, they get installed successfully when I start the server again, which is what I want. However, the script starts throwing Cannot find module errors when I uninstall the first or first two modules of the list that use the function requireOrInstall. That's right, the errors only occur when the script has to install either the first or the first two modules, not when only the second module needs installing.
In this example, the error will be thrown when I uninstall find-free-port, unless I move its require at least one spot down ¯\_(• _ •)_/¯
I've also tried adding a delay directly after the synchronous install to give it a little more breathing time with the following two lines:
var until = new Date().getTime() + 1000
while (new Date().getTime() < until) {}
The pause was there. It didn't fix anything.
#velocityzen came with the idea to check the cache, which I've now added to the script. It doesn't show anything out of the ordinary.
#vaughan's comment on another question noted that this exact error occurs when requiring a module twice. I've changed the script to use require.resolve(), but the error still remains.
Does anybody know what could be causing this?
Edit
Since the question has been answered, I'm posting the one-liner (139 characters!). It doesn't globally define child_modules, has no last try-catch and doesn't log anything in the console:
const req=async m=>{let r=require;try{r.resolve(m)}catch(e){r('child_process').execSync('npm i '+m);await setImmediate(()=>{})}return r(m)}
The name of the function is req() and can be used like in #alex-rokabilis' answer.

It seems that the require operation after an npm install needs a certain delay.
Also the problem is worse in windows, it will always fail if the module needs to be npm installed.
It's like at a specific event snapshot is already known what modules can be required and what cannot. Probably that's why require.cache was mentioned in the comments. Nevertheless I suggest you to check the 2 following solutions.
1) Use a delay
const cp = require("child_process");
const requireOrInstall = async module => {
try {
require.resolve(module);
} catch (e) {
console.log(`Could not resolve "${module}"\nInstalling`);
cp.execSync(`npm install ${module}`);
// Use one of the two awaits below
// The first one waits 1000 milliseconds
// The other waits until the next event cycle
// Both work
await new Promise(resolve => setTimeout(() => resolve(), 1000));
await new Promise(resolve => setImmediate(() => resolve()));
console.log(`"${module}" has been installed`);
}
console.log(`Requiring "${module}"`);
try {
return require(module);
} catch (e) {
console.log(require.cache);
console.log(e);
}
}
const main = async() => {
const http = require("http");
const path = require("path");
const fs = require("fs");
const ffp = await requireOrInstall("find-free-port");
const express = await requireOrInstall("express");
const socket = await requireOrInstall("socket.io");
}
main();
await always needs a promise to work with, but it's not needed to explicitly create one as await will wrap whatever it is waiting for in a promise if it isn't handed one.
2) Use a cluster
const cp = require("child_process");
function requireOrInstall(module) {
try {
require.resolve(module);
} catch (e) {
console.log(`Could not resolve "${module}"\nInstalling`);
cp.execSync(`npm install ${module}`);
console.log(`"${module}" has been installed`);
}
console.log(`Requiring "${module}"`);
try {
return require(module);
} catch (e) {
console.log(require.cache);
console.log(e);
process.exit(1007);
}
}
const cluster = require("cluster");
if (cluster.isMaster) {
cluster.fork();
cluster.on("exit", (worker, code, signal) => {
if (code === 1007) {
cluster.fork();
}
});
} else if (cluster.isWorker) {
// The real work here for the worker
const http = require("http");
const path = require("path");
const fs = require("fs");
const ffp = requireOrInstall("find-free-port");
const express = requireOrInstall("express");
const socket = requireOrInstall("socket.io");
process.exit(0);
}
The idea here is to re-run the process in case of a missing module. This way we fully reproduce a manual npm install so as you guess it works! Also it seems more synchronous rather the first option, but a bit more complex.

I think your best option is either:
(ugly) to install package globally, instead of locally
(best solution ?) to define YOUR new 'package repository installation', when installing, AND when requiring
First, you may consider using the npm-programmatic package.
Then, you may define your repository path with something like:
const PATH='/tmp/myNodeModuleRepository';
Then, replace your installation instruction with something like:
const npm = require('npm-programmatic');
npm.install(`${module}`, {
cwd: PATH,
save:true
}
Eventually, replace your failback require instruction, with something like:
return require(module, { paths: [ PATH ] });
If it is still not working, you may update the require.cache variable, for instance to invalide a module, you can do something like:
delete require.cache[process.cwd() + 'node_modules/bluebird/js/release/bluebird.js'];
You may need to update it manually, to add information about your new module, before loading it.

cp.execSync is an async call so try check if the module is installed in it's call back function. I have tried it, installation is clean now:
const cp = require('child_process')
function requireOrInstall (module) {
try {
require.resolve(module)
} catch (e) {
console.log(`Could not resolve "${module}"\nInstalling`)
cp.execSync(`npm install ${module}`, () => {
console.log(`"${module}" has been installed`)
try {
return require(module)
} catch (e) {
console.log(require.cache)
console.log(e)
}
})
}
console.log(`Requiring "${module}"`)
}
const http = require('http')
const path = require('path')
const fs = require('fs')
const ffp = requireOrInstall('find-free-port')
const express = requireOrInstall('express')
const socket = requireOrInstall('socket.io')
When node_modules not available yet :
When node_modules available already:

Related

What is a VS Code Command to run a python file?

I have made an extension that opens a file dialog. What I would like to do is after the file is selected, I want a python file to run. What I need is the VS Code command to run a file (or perhaps a python file specifically?).
here is a working example where the command I use is a command that comments the selected line in the currently active editor. It works perfectly so I know this structure is generally correct. I just need the right command to replace the comment line command.
below is the code in questions with the working command I mentioned above. I found it using this resource: where I found the comment line command
// The module 'vscode' contains the VS Code extensibility API
// Import the module and reference it with the alias vscode in your code below
const { ChildProcess } = require('child_process');
const vscode = require('vscode');
const { execFile } = require('node:child_process');
const { stdout, stderr } = require('process');
// this method is called when your extension is activated
// your extension is activated the very first time the command is executed
/**
* #param {vscode.ExtensionContext} context
*/
function activate(context) {
let disposable = vscode.commands.registerCommand('fileDialog.openFile', function () {
const options = {
canSelectMany: false,
openLabel: 'Open'
};
vscode.window.showOpenDialog(options).then(fileUri => {
if (fileUri && fileUri[0]) {
console.log('Selected file: ' + fileUri[0].fsPath);
vscode.commands.executeCommand('python.execInInterminal');
}
});
});
context.subscriptions.push(disposable);
}
// this method is called when your extension is deactivated
function deactivate() {}
module.exports = {
activate,
deactivate
}
You can go for child_process' exec or spawn if you only need to run the Python script(s).
If you really prefer to rely on the Python extension, then you'll need to at least feed the script's Uri into the executeCommand's argument - its the 2nd part of what you found.
function activate(context) {
let disposable = vscode.commands.registerCommand('fileDialog.openFile', function () {
const options = {
canSelectMany: false,
openLabel: 'Open',
filters: {
'Python script': ['py']
}
};
vscode.window.showOpenDialog(options).then((fileUris) => {
// This will always get an array of Uris regardless of canSelectMany being false
fileUris?.forEach(async (fileUri) => {
console.log(`Selected file: ${fileUri.fsPath}`);
await vscode.commands.executeCommand('python.execInInterminal', fileUri);
});
});
});
context.subscriptions.push(disposable);
}
If you want to handle more things, you can refer to the thenable unittest code they included in the repo:
https://github.com/microsoft/vscode-python/blob/main/src/test/smoke/runInTerminal.smoke.test.ts#L46-L48

Check if child_process can run a command in NodeJS

how can I check if child_process can run a command?
'echo' is a valid command that can be run in a terminal, but 'echoes' is not one. For example, if I do this
const cp = require('child_process')
cp.exec('echo hello')
it will work.
If I do this, though
const cp = require('child_process')
cp.exec('echoes hello') //notice how it is echoes instead of echo
it will just error, but maybe the user has a program that adds 'echoes' to a terminal, and in that case, it would be able to run, but if it errors it will just exit out of the process and I won't be able to check if it works.
Is there any way to do this? Thank you so much in advance!
You have to manually loop through dirs in $PATH env & perform look up on those directory.
eg: $PATH is set to /bin:/usr/local/bin then you have to perform
fs.access('/bin/' + command, fs.constants.X_OK)
and
fs.access('/usr/local/bin/' + command, fs.constants.X_OK)
solution would look like this.
const { constants: fsconsts } = require('fs')
const fs = require('fs/promises')
const path = require('path')
const paths = process.env.PATH.split(':')
async function isExecutable(command) {
const cases = []
for (const p of paths) {
const bin = path.join(p, command)
cases.push(fs.access(bin, fsconsts.X_OK)) // X_OK is bit flag which makes sure file is executable
}
await Promise.any(cases)
return command
}
const found = (bin) => console.log('found', bin)
const notfound = (errors) => {
console.log('not found or not executable')
// console.error(errors)
}
// passes
isExecutable('echo').then(found).catch(notfound)
isExecutable('node').then(found).catch(notfound)
// fails
isExecutable('shhhhhh').then(found).catch(notfound)
isExecutable('echoes').then(found).catch(notfound)
NOTE: I think my solution works only on *nix based OSs

Hanging promise canceled

I'm trying to set Cloudflare's workers to track the circulation of some ERC20 tokens as an exercise to learn web3 and wasm. Thought it could be simple enough, but about 90% of the time so far has been trying to solve this elusive error
A hanging Promise was canceled. This happens when the worker runtime is waiting for a Promise from JavaScript to resolve but has detected that the Promise cannot possibly ever resolve because all code and events related to the Promise's request context have already finished.
I look for additional information online, but it seems my error is from a different type(?).
Here's a simple snippet of code to reproduce.
mod erc20_abi;
use erc20_abi::ERC20_ABI;
use cfg_if::cfg_if;
use ethers::{
contract::Contract,
core::{abi::Abi, types::Address},
prelude::{AbiError, U256},
providers::{Http, Provider},
};
use num_format::{Locale, ToFormattedString};
use std::convert::TryFrom;
use wasm_bindgen::prelude::*;
cfg_if! {
// When the `wee_alloc` feature is enabled, use `wee_alloc` as the global
// allocator.
if #[cfg(feature = "wee_alloc")] {
extern crate wee_alloc;
#[global_allocator]
static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT;
}
}
#[wasm_bindgen]
pub async fn handle() -> String {
let web3_ethereum = Provider::<Http>::try_from(WEB3_URL_ETHEREUM).unwrap();
let abi: Abi = serde_json::from_str(ERC20_ABI).unwrap();
let token_contract_ethereum = Contract::new(parse_address(ADDRESS_ETH),
abi, web3_ethereum);
let convert_wei_to_decimal = |bignumber: U256| -> String {
(bignumber.as_u128() / u128::pow(10, 18)).to_formatted_string(&Locale::en)
};
// I believe this is the problem, since just returning a String works fine.
let total_supply_ethereum = token_contract_ethereum
.method::<_, U256>("totalSupply", ())
.unwrap()
.call()
.await
.unwrap();
convert_wei_to_decimal(total_supply_ethereum)
}
fn parse_address(address: &str) -> Address {
address.parse::<Address>().unwrap()
}
This is the worker/workers.js file
addEventListener('fetch', (event) => {
event.respondWith(handleRequest(event.request))
})
const { handle } = wasm_bindgen;
const instance = wasm_bindgen(wasm);
/**
* Fetch and log a request
* #param {Request} request
*/
async function handleRequest(request) {
await instance;
const output = await handle();
let res = new Response(output, { status: 200 });
res.headers.set('Content-type', 'text/html');
return res;
}
Cargo.toml
[package]
name = "circulating-supply"
version = "0.1.0"
license = "GPL-3.0-or-later"
edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[lib]
crate-type = ["cdylib", "rlib"]
[profile.release]
opt-level = 's' # Optimize for size.
lto = true
panic = "abort"
codegen-units = 1
[dependencies]
ethers = { git = "https://github.com/gakonst/ethers-rs" }
serde_json = "1.0.68"
num-format = "0.4.0"
cfg-if = "1.0.0"
wee_alloc = { version = "0.4.5", optional = true }
wasm-bindgen = "0.2.78"
wasm-bindgen-futures = "0.4.28"
js-sys = "0.3.55"
wrangler dev will compile it fine, but going to http://127.0.0.1:8787 will result in Error 1101
In my case a dependency used sth. not available in wasm runtime.
I guess ethers cryptography dependencies also depend on sth. like getrandom.
Adding this to Cargo.toml solved my issue.
[target.wasm32-unknown-unknown.dependencies]
getrandom = { version = "0.1", features = ["wasm-bindgen"] }
This will force your dependencies based on getrandom use the wasm features of getrandom.

Having trouble understanding multiple awaits in async functions

I wrote this function to generate png files from some svg files across a few directories. I was doing the below functionality synchronously and it was working as expected (same code as below but with readFileSync), but was told to re-do it to use only promisified fs functions.
The current code skips a couple files in both groupA and groupB, plus its swapping widths. For example, I've noticed the conversion function wont generate for svg1 of dirB, but will generate for svg1 of dirA though it has incorrect width that matches svg1 of dirB.
Most files convert correctly, but a handful don't. My guess is its a timing issue, so how do I fix that while still keeping the fs functionality all promisified?
const { createConverter } = require('convert-svg-to-png');
const fs = require('fs');
const path = require('path');
const util = require('util');
const readdir = util.promisify(fs.readdir);
const readFile = util.promisify(fs.readFile);
async function convertSvgFiles(dirPath) {
const converter = createConverter();
try {
const files = await readdir(dirPath);
for (let file of files) {
const currentFile = path.join(dirPath, fil);
const fileContents = await readFile(currentFile);
const fileWidth = fileContents.toString('utf8').match(\*regex capture viewbox width*\);
await converter.convertFile(currentFile, { width: fileWidth });
}
} catch (err) {
console.warn('Error while converting a file to png', '\n', err);
} finally {
await converter.destroy();
}
}
['dirA', 'dirB', 'dirC'].map(dir => convertSvgFiles(`src/${dir}`));
Your code looks pretty good. I do not see anything obvious that would cause the behavior you describe - you await each promise within the function. You are not using global variables.
I'll say that this line:
['dirA', 'dirB', 'dirC'].map(dir => convertSvgFiles(`src/${dir}`));
ends up running your function on all 3 directories in parallel, with 3 converter instances. Assuming there are no parallel-related bugs in converter, this should not cause any issues.
But just for grins, try changing that to:
async function run() {
for (let d of ['dirA', 'dirB', 'dirC']) {
await convertSvgFiles(`src/${d}`)
}
}
run()
to force the folders to be scanned sequentially. If this resolves the issue, then there's a bug within convert-svg-to-png.

truffle test will only run one contract

I have an instability issue.
I'm using openzeppelin-test-helpers, and first had an issue with typescript saying Could not find a declaration file for module '#openzeppelin/test-helpers'.. This issue was solved by creating a .d.ts file containing declare module "#openzeppelin/test-helpers";.
However, adding this created a new problem, which is that now, most of the time, only one file is being run by rm -rf build && truffle test (I guess this is similar to truffle test --reset).
I got 2 test files. The first one looks like:
require("chai")
.use(require("chai-as-promised"))
.should();
const EventHandler = artifacts.require("EventHandler");
const { expectRevert } = require("#openzeppelin/test-helpers");
contract("EventHandler", function([_, superAdmin0, admin0, device0]) {
beforeEach(async function() {
this.eventHandler = await EventHandler.new(superAdmin0);
});
describe("Initial tests", function() {
it("should print hello", async function() {
await this.eventHandler
.printHello()
.should.eventually.equal("Hello");
});
});
});
The second one looks like:
require("chai")
.use(require("chai-as-promised"))
.should();
const { expectRevert } = require("#openzeppelin/test-helpers");
const EventHandler = artifacts.require("EventHandler");
contract("Roles", function([_, superAdmin0, superAdmin1, admin0, device0]) {
beforeEach(async function() {
this.EventHandler = await EventHandler.new(superAdmin0);
});
it("...should work", async function() {});
});
When I comment the content of one file, or just what's inside contract(..., {}), the other file works just fine and the tests pass successfully.
However, whenever I leave those 2 files uncommented, I get a massive error:
Error: Returned values aren't valid, did it run Out of Gas?
Of course, resetting ganache-cli didn't solve anything...
Does anyone know where it could come from?

Categories