I have some code like so:
export async function handleRefresh() {
if (!existsSync('postr.toml')) fail('not a postr directory');
const posts = expandGlob('posts/*');
for await (const post of posts) {
if (!post.isDirectory) {
console.warn('warning: non-folder found in posts directory');
continue;
}
let {parsedFrontMatter, contents} = extractFrontMatter(await read(post.path + '/post.md'));
const adapters = parsedFrontMatter.adapters ?? [];
if (!parsedFrontMatter) {
fail('no frontmatter for ' + post.path);
continue;
}
if (!Array.isArray(adapters)) {
fail('adapters is not an array');
continue;
}
if (isValidFrontMatter(parsedFrontMatter)) {
fail('frontmatter is not valid');
continue;
}
adapters.forEach(async (adapter: string) => {
const adapterPlugins = parseToml(await read('postr.toml')).adapterPlugins ?? {};
if (!isObject(adapterPlugins)) {
fail('adapterPlugins in the configuration is not an object');
return;
}
const adapterPath = adapterPlugins[adapter];
if (!adapterPath) {
console.warn('warn: an adapter was set' +
'but the corresponding plugin was not configured in `postr.toml`. Skipping');
return;
}
if (!('path' in <any>adapterPath)) {
fail(`adapter ${adapter} does not have a path`);
return;
}
import((<any>adapterPath).path)
.then(async module => {
const action = getActionForPost(parsedFrontMatter);
if (module[action]) {
await module[action](contents, parsedFrontMatter, (<any>adapterPath).config, {
updateFrontMatter(newData: {[x: string]: any}) {
parsedFrontMatter = Object.assign(parsedFrontMatter, newData);
},
mapID(remote: string | number) {
addMapping(parsedFrontMatter.id as string, remote.toString(), adapter);
}
})
} else {
console.warn(`Adapter ${adapter} does not support action \`${action}\``);
return;
}
writeFinalContents(parsedFrontMatter, contents, post.path)
})
.catch(error => fail(`could not run adapter because of ${error.name}: ${error.message}`));
});
}
}
Huge function.
There are a lot of these necessary if checks. 3/4 of the function is if checks, you could say. I want some advice on how I could refactor these statements.
As you can see the checks are not always the same, there are some different checks going on there.
EDIT: I've added real code.
Related
I am tring to download around 20.000 images from a website by scraping it.
(I am authorized by the owner)
The path of the image is composed like this:
VolumeId/ChapterId/PageId.jpg
There are around 100 volumes, and every volume has x chapters, and every chapter has y pages.
In the database I have stored for every volume the number of chapters, but i don't have
the number of pages, so I have to navigate to the image url and check if it exists.
I know for sure that there are less than 30 pages per chapter so I did something like this:
let exists = true;
for (let i = 0; i < 30 && exists; i++) {
fetch(`imgUrl/${i}.jpg`)
.then(data => {
if (data.ok)
return data.arrayBuffer();
else
exists = false;
.then(arrayBuffer => {
if (exists) {
let buffer = Buffer.from(arrayBuffer );
if (!fs.existsSync(path.join(__dirname, imgPath))) {
fs.mkdirSync(path.join(__dirname, imgPath), {
recursive: true,
});
}
fs.writeFile(
path.join(__dirname, imgPath + "/" + img + ".jpg"),
buffer,
(err) => {
if (err) throw err;
}
);
}
});
}
The problem:
The problem is that the loop does not wait for the image to be fetched and saved locally.
I have tried with async/await and promises (I think I have implemented them wrong)
Is there a better way to download a large quantity of data? Maybe with streams?
It can be a little bit complicated to implement your code with just async/await and at the same time assure the "exists" condition between iterations, I suggest you use a class that implements an async iterator, refer to the official documentation for more details, the following code achieve what you are looking for (note: the code snippet you provided didn't show where "imgPath" is coming from so just fix my code accordingly) :
class FetchImages {
constructor(urls) {
this.urls = urls;
this.index = 0;
}
[Symbol.asyncIterator]() {
const urlsIterator = this.urls[Symbol.iterator]();
return {
async next() {
if (++index == 30) {
return {
done: true
};
}
const iteratorResult = urlsIterator.next();
if (iteratorResult.done) {
return { done: true };
}
const url = iteratorResult.value;
try {
let response = await fetch(url);
let data;
if (response.status == 200) {
data = await response.arrayBuffer();
} else {
// equivalent to exists == false, exit condition of the iterator
return {
done: true
};
};
let buffer = Buffer.from(data);
if (!fs.existsSync(path.join(__dirname, imgPath))) {
fs.mkdirSync(path.join(__dirname, imgPath), {
recursive: true,
});
}
fs.writeFileSync(
path.join(__dirname, imgPath),
buffer,
);
return {
done: false,
value: imgPath
};
} catch (err) {
return {
done: false,
value: err.message
};
}
}
}
}
}
(async function () {
const fetchImages = new FetchImages(urls);
for await (const fetchImage of fetchImages) {
// check status of each fetch
console.log(fetchImage);
}
})();
Function validateBusinessId results in void variable businessExists. I am trying to assign different database paths depending on whether the business exists in the database or if the uploaded file is a master file. Function validateBusiness produces void result. Not sure how to fix it. Where did I go wrong?
async function validateBusinessId(businessId: string) {
db.collection('business').doc(businessId).get()
.then((docSnapshot) => {
if (docSnapshot.exists) {
return true
} else {
return false
}
})
}
async function getDatabase(fileName: string, businessId: string) {
const businessExists = await validateBusinessId(businessId)
if ((fileName) && (fileName?.includes('masterFile'))) {
console.log('A new Master database file has been detected.')
return db.collection('products')
} else if ((businessId) && (businessExists)) {
// If the business exists, return business path
console.log('A new database file has been detected for business: ' + businessId)
return db.collection('business').doc(businessId).collection('products')
} else {
return null
}
}
Help is highly appreciated!
Ok, so I found that I handled the promise of a query result incorrectly. Here's the code that worked for me.
async function validateBusinessId(businessId: string) {
const databaseDocumentSnapshot = await db.collection('business').doc(businessId).get()
if (databaseDocumentSnapshot.exists) {
return true
} else {
return false
}
}
async function getDatabase(fileName: string, businessId: string) {
const businessExists = await validateBusinessId(businessId)
if ((fileName) && (fileName?.includes('masterFile'))) {
console.log('A new Master database file has been detected.')
return db.collection('products')
} else if ((businessId) && (businessExists)) {
return db.collection('business').doc(businessId).collection('products')
} else {
console.log('No business with such ID exists: ' + businessId);
return null
}
}
I have an Azure Function that inserts from JSON-LD into my GraphDB - however what I'm finding is my GraphDB keeps crashing, because the function is sending too many insert requests.
I've set "maxConcurrentRequests": 1 and "maxOutstandingRequests": 1 however it still doesn't seem to wait and process one at a time.
Could someone please explain why this so.
export async function onTrigger(context: Context, documents: Resource[] | null): Promise<void> {
if (documents == null) { return }
documents.forEach(async function (value: any) {
if ("metadata" in value) { } else { return; }
let song: MyDocument = value.metadata.songs.song;
// Create the JSON-LD object using the song object from above
let jsonld = ...
let nQuads = await jsonldParser.toRDF(jsonld, {format: 'application/n-quads'});
let insertQuery = "INSERT DATA {" + nQuads + "}";
try {
let res = await axios.post('http://localhost:7200/mygraphdb', "update="+encodeURIComponent(insertQuery))
if (res.status === 204) {
console.log(`All Done!`);
}
} catch (error) {
console.log(`Error! ${song.id}`);
}
});
}
I am working on a Resource Manager and some jobs must be atomic, because there may be race condition.
I see a solution is provided in a similar post
Javascript semaphore / test-and-set / lock?
which says implement a basic integer semaphore, just add the variable into the DOM and lock/unlock it and make sure your functions keep checking it, else timeout them
Can someone help me how can I implement it for below code ? and How can I add a variable to DOM ?
In the code below retrievePartners methods returns partners based on what user asked in capabilities object, and retrievePartners method may have been called same time from another user, asking for same partners (browsers.) So this method should be atomic, means should deal only with one capabilities at a same time.
async retrievePartners (capabilities) {
const appropriatePartners = { chrome: [], firefox: [], safari: [], ie: [] }
const partners = await this.getAllPartners()
allTypeNumber = 0
// first check if there is available appropriate Partners
Object.keys(capabilities.type).forEach(key => {
let typeNumber = parseInt(capabilities.type[key])
allTypeNumber = allTypeNumber + typeNumber
for (let i = 0; i < typeNumber; i++) {
partners.forEach((partner, i) => {
if (
key === partner.value.type &&
partner.value.isAvailable &&
appropriatePartners[key].length < typeNumber
) {
appropriatePartners[key].push(partner)
}
})
if (appropriatePartners[key].length < typeNumber) {
throw new Error(
'Sorry there are no appropriate Partners for this session'
)
}
}
})
if (allTypeNumber === 0) {
throw new Error('Please mention at least 1 type of browser !')
} else {
Object.keys(appropriatePartners).forEach(key => {
appropriatePartners[key].forEach(partner => {
this.instructorPeer.set('/partners/' + partner.id + '/states/', {
isAvailable: false
})
})
})
return appropriatePartners
}
}
getAllPartners method
async getAllPartners (capabilities) {
const partners = []
const paths = await this.instructorPeer.get({
path: { startsWith: '/partners/' }
})
paths.forEach((path, i) => {
if (path.fetchOnly) {
let obj = {}
obj.value = path.value
obj.id = path.path.split('/partners/')[1]
obj.value.isAvailable = paths[i + 1].value.isAvailable
partners.push(obj)
}
})
return partners
}
Here is how I call the method
async function startTest () {
const capabilities = {
type: {
chrome: 1
}
}
test('Example test', async t => {
try {
session = await workout.createSession(capabilities)
const partners = session.partners
const partner = partners.chrome[0]
...
I'm writing an VS Code extension to help migrating React.createClass to class extends React.Component. The problem here was, I could not get vscode.commands.executeCommand('vscode.executeFormatDocumentProvider', ...) to work.
Note that the code below is pure JavaScript, but not TypeScript.
function activate(context) {
context.subscriptions.push(vscode.commands.registerCommand('migrate-to-react-es6-class', () => {
const editor = vscode.window.activeTextEditor
const document = editor.document
try {
const originalCode = document.getText()
const modifiedCode = 'do something and return new code'
if (originalCode === modifiedCode) {
vscode.window.showInformationMessage('Nothing is to be migrated.')
} else {
editor.edit(edit => {
const editingRange = document.validateRange(new vscode.Range(0, 0, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER))
edit.replace(editingRange, modifiedCode)
})
if (document.isUntitled === false) {
vscode.commands.executeCommand('vscode.executeFormatDocumentProvider', document.uri, { insertSpaces: true, tabSize: 2 })
}
}
} catch (error) {
vscode.window.showErrorMessage(error.message)
console.error(error)
}
}))
}
After 3.25 years, you've probably figured this out by now, but for the record, I assume you hung a .then() on the editor.edit() and then moved the executeCommand to within the then(), right?
editor.edit(edit => {
const editingRange = ...
edit.replace(editingRange, modifiedCode)
}).then(editWorked => {if (editWorked && !document.isUntitled) {
vscode.commands.executeCommand('vscode.executeFormatDocumentProvider', ...) })
You must apply returned edits
private async formatDocument(): Promise<void> {
const docUri = this.textEditor.document.uri;
const textEdits = (await vscode.commands.executeCommand(
'vscode.executeFormatDocumentProvider',
docUri,
)) as vscode.TextEdit[];
const edit = new vscode.WorkspaceEdit();
for (const textEdit of textEdits) {
edit.replace(docUri, textEdit.range, textEdit.newText);
}
await vscode.workspace.applyEdit(edit);
}
I implemented it directly in the extension.ts:
commands.registerCommand(constants.commands.formatDocument, async () => {
const docUri = editor?.document.uri;
const textEdits: TextEdit[] | undefined = await commands.executeCommand(
'vscode.executeFormatDocumentProvider',
docUri
);
if (textEdits && docUri) {
const edit = new WorkspaceEdit();
for (const textEdit of textEdits) {
edit.replace(docUri, textEdit.range, textEdit.newText);
}
await workspace.applyEdit(edit);
}
});
constants.commands.formatDocument gets the value after parsing my package.json