hi somebody can help me to create a custom command on cypress to use this async function
var jimp = require("jimp");
var qrCode = require('qrcode-reader');
var fs = require('fs')
var path = require("path")
const readQRCode = async (fileName) => {
const filePath = path.join(__dirname, fileName)
try {
if (fs.existsSync(filePath)) {
const img = await jimp.read(fs.readFileSync(filePath));
const qr = new qrCode();
const value = await new Promise((resolve, reject) => {
qr.callback = (err, v) => err != null ? reject(err) : resolve(v);
qr.decode(img.bitmap);
});
return value.result;
}
} catch (error) {
return error.message
}
}
I am trying to save to json the values returned from indeed api. I use indeed-scraper code from github https://github.com/rynobax/indeed-scraper
My code:
... required files ...
const parsedResults = []
indeed.query(queryOptions).then(response => {
response.forEach((res,i) => {
setTimeout(function(){
let url = res.url
let resultCount = 0
console.log(`\n Scraping of ${url} initiated...\n`)
const getWebsiteContent = async (url) => {
try {
const response = await axios.get(url)
const $ = cheerio.load(response.data)
...get scraped data...
parsedResults.push(metadata)
} catch (error) {
exportResults(parsedResults)
console.error(error)
}
}
getWebsiteContent(url)
}
, i*3000);
});
});
const outputFile = 'data.json'
const fs = require('fs');
const exportResults = (parsedResults) => {
fs.writeFile(outputFile, JSON.stringify(parsedResults, null, 4), (err) => {
if (err) {
console.log(err)
}
console.log(`\n ${parsedResults.length} Results exported successfully to ${outputFile}\n`)
})
}
parsedResults is not accessible in last portion of script, so to save as json file.
Any help appreciated!
The script used when trying to get contents from the csv stored in the s3 bucket
const mysql = require("mysql");
const fs = require("fs");
const { google } = require("googleapis");
const AWS = require("aws-sdk");
const client = new AWS.SecretsManager({ region: "eu-west-1" });
const analyticsreporting = google.analyticsreporting("v4");
const csv = require('ya-csv')
const fastCsv = require('fast-csv')
const s3 = new AWS.S3();
const getParams = {
Bucket: 'data',
Key: 'athena_test/nameplate.csv'
};
exports.handler = async (context, event) => {
const data = await s3.getObject(getParams, function (err, data){
if(err){console.log("ERROR: ",err)}
else {return data}
})
console.log(data.Body)
}
the console log returns undefined rather than the contents of the csv
Hey you can try this one:-
const csv = require('#fast-csv/parse');
const s3Stream = await s3.getObject(params).createReadStream();
const data = await returnDataFromCSV();
console.log(data.Body);
const returnDataFromCSV =()=> {
let promiseData = new Promise((resolve, reject) => {
const parser = csv
.parseStream(csvFile, { headers: true })
.on("data", (data) => {
console.log('Parsed Data:-', data);
})
.on("end", ()=> {
resolve("CSV finished here");
})
.on("error",()=> {
reject("if failed");
});
});
try {
return await promiseData;
} catch (error) {
console.log("Get Error: ", error);
return error;
}
}
CreateStream: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Request.html#createReadStream-property
Cloud Functions for Firebase has this nice sample where they create a thumbnail for each uploaded image. This is done by making use of ImageMagick.
I tried to convert the sample to convert PDFs to images. This is something ImageMagick can do, but I can't make it work with Cloud Functions for Firebase. I keep getting a code 1 error:
ChildProcessError: `convert /tmp/cd9d0278-16b2-42be-aa3d-45b5adf89332.pdf[0] -density 200 /tmp/cd9d0278-16b2-42be-aa3d-45b5adf89332.pdf` failed with code 1
at ChildProcess.<anonymous> (/user_code/node_modules/child-process-promise/lib/index.js:132:23)
at emitTwo (events.js:106:13)
at ChildProcess.emit (events.js:191:7)
at maybeClose (internal/child_process.js:877:16)
at Socket.<anonymous> (internal/child_process.js:334:11)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at Pipe._handle.close [as _onclose] (net.js:498:12)
Of course one possibility is that converting PDFs are simply not supported.
const functions = require('firebase-functions');
const gcs = require('#google-cloud/storage')();
const spawn = require('child-process-promise').spawn;
// [END import]
// [START generateThumbnail]
/**
* When an image is uploaded in the Storage bucket We generate a thumbnail automatically using
* ImageMagick.
*/
// [START generateThumbnailTrigger]
exports.generateThumbnail = functions.storage.object().onChange(event => {
// [END generateThumbnailTrigger]
// [START eventAttributes]
const object = event.data; // The Storage object.
const fileBucket = object.bucket; // The Storage bucket that contains the file.
const filePath = object.name; // File path in the bucket.
const contentType = object.contentType; // File content type.
const resourceState = object.resourceState; // The resourceState is 'exists' or 'not_exists' (for file/folder deletions).
// [END eventAttributes]
// [START stopConditions]
// Exit if this is triggered on a file that is not an image.
if (!contentType.startsWith('application/pdf')) {
console.log('This is not a pdf.');
return;
}
// Get the file name.
const fileName = filePath.split('/').pop();
// Exit if the image is already a thumbnail.
if (fileName.startsWith('thumb_')) {
console.log('Already a Thumbnail.');
return;
}
// Exit if this is a move or deletion event.
if (resourceState === 'not_exists') {
console.log('This is a deletion event.');
return;
}
// [END stopConditions]
// [START thumbnailGeneration]
// Download file from bucket.
const bucket = gcs.bucket(fileBucket);
const tempFilePath = `/tmp/${fileName}`;
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
console.log('Pdf downloaded locally to', tempFilePath);
// Generate a thumbnail of the first page using ImageMagick.
return spawn('convert', [tempFilePath+'[0]' ,'-density', '200', tempFilePath]).then(() => {
console.log('Thumbnail created at', tempFilePath);
// Convert pdf extension to png
const thumbFilePath = filePath.replace('.pdf', 'png');
// Uploading the thumbnail.
return bucket.upload(tempFilePath, {
destination: thumbFilePath
});
});
});
// [END thumbnailGeneration]
});
Node modules can install native code that is in the same directory as the Cloud Function's source code. I found that some node libraries on github that do this for ghostscript which is a very useful library for PDF processing:
Node library that wraps Ghostscript command line:
https://github.com/sina-masnadi/node-gs
Compiled Ghostscript which is
used via git submodule:
https://github.com/sina-masnadi/lambda-ghostscript
I put lambda-ghostscript into a sub-directory of my functions directory, then add the node-gs as a dependency in my package file like this:
{
"name": "functions",
"dependencies": {
"#google-cloud/storage": "^1.3.1",
"child-process-promise": "^2.2.1",
"firebase-admin": "~5.4.0",
"firebase-functions": "^0.7.2",
"gs": "https://github.com/sina-masnadi/node-gs/tarball/master"
}
}
Then in my index.js file I can just require the node library to easily use ghostscript from JavaScript. Here's the complete code for the Cloud Function that uses a Google Cloud Storage trigger:
const functions = require('firebase-functions');
const gcs = require('#google-cloud/storage')();
const spawn = require('child-process-promise').spawn;
const path = require('path');
const os = require('os');
const fs = require('fs');
var gs = require('gs');
exports.makePNG = functions.storage.object().onChange(event => {
// ignore delete events
if (event.data.resourceState == 'not_exists') return false;
const filePath = event.data.name;
const fileDir = path.dirname(filePath);
const fileName = path.basename(filePath);
const tempFilePath = path.join(os.tmpdir(), fileName);
if (fileName.endsWith('.png')) return false;
if (!fileName.endsWith('.pdf')) return false;
const newName = path.basename(filePath, '.pdf') + '.png';
const tempNewPath = path.join(os.tmpdir(), newName);
// // Download file from bucket.
const bucket = gcs.bucket(event.data.bucket);
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
console.log('Image downloaded locally to', tempFilePath);
return new Promise(function (resolve, reject) {
gs()
.batch()
.nopause()
.option('-r' + 50 * 2)
.option('-dDownScaleFactor=2')
.executablePath('lambda-ghostscript/bin/./gs')
.device('png16m')
.output(tempNewPath)
.input(tempFilePath)
.exec(function (err, stdout, stderr) {
if (!err) {
console.log('gs executed w/o error');
console.log('stdout',stdout);
console.log('stderr',stderr);
resolve();
} else {
console.log('gs error:', err);
reject(err);
}
});
});
}).then(() => {
console.log('PNG created at', tempNewPath);
// Uploading the thumbnail.
return bucket.upload(tempNewPath, {destination: newName});
// Once the thumbnail has been uploaded delete the local file to free up disk space.
}).then(() => {
fs.unlinkSync(tempNewPath);
fs.unlinkSync(tempFilePath);
}).catch((err) => {
console.log('exception:', err);
return err;
});
});
Here's the project on github: https://github.com/ultrasaurus/ghostscript-cloud-function
Disclaimer: This is using compiled native code and I verified experimentally that works for this case, so it is probably fine. I didn't look into the specific compile options and validate if they exactly correct for the Cloud Functions environment.
WORKING SOLUTION
Thank you #Ultrasaurus for pointing out this approach! However, for me it did not work and in your Github repo your also stated I haven't tested them. I modified your solution a little bit and got the following code, which is 100% working for me:
{
"dependencies": {
"#google-cloud/firestore": "^4.4.0",
"#google-cloud/storage": "^5.3.0",
"ghostscript": "https://github.com/musubu/node-ghostscript/tarball/master",
"pdf-image": "^2.0.0",
"rimraf": "^3.0.2",
"uuid": "^8.3.1"
}
}
The function is triggered by a Firestore event:
const Storage = require('#google-cloud/storage')
const fs = require('fs')
const rimraf = require('rimraf')
const os = require('os')
const gs = require('ghostscript')
const GOOGLE_PROJECT_ID = 'MY_GOOGLE_PROJECT_ID'
const GOOGLE_STORAGE_BUCKET_NAME = 'MY_GOOGLE_STORAGE_BUCKET_NAME'
const storage = new Storage.Storage({
projectId: GOOGLE_PROJECT_ID
})
exports.createImage = async (event) => {
let {
appointment,
name
} = event.value.fields
name = getFileName(name.stringValue)
appointment = appointment.stringValue
console.log(`Processing document ${name} in appointment ${appointment}`)
const tempDir = createTempDir(appointment)
const tmpDocumentPath = await downloadPdf(tempDir, name, appointment)
const imagePath = await convertPdfToImage(tmpDocumentPath)
await uploadImage(imagePath, appointment)
deleteDir(tempDir)
}
function getFileName (name) {
const nameParts = name.split('/')
return nameParts[nameParts.length - 1]
}
function createTempDir (appointment) {
const tempDir = `${os.tmpdir()}/${appointment}_${Math.random()}`
fs.mkdirSync(tempDir)
console.log(`Created dir ${tempDir}`)
return tempDir
}
async function downloadPdf (tempDir, name, appointment) {
const destination = `${tempDir}/${name}`
await storage.bucket(GOOGLE_STORAGE_BUCKET_NAME).file(`${appointment}/${name}`).download({ destination })
console.log(`Successfully downloaded document ${name}`)
return destination
}
async function convertPdfToImage (pdfPath) {
const imagePath = pdfPath.replace('pdf', 'png')
return new Promise(function (resolve, reject) {
try {
gs()
.batch()
.nopause()
.device('png16m')
.output(imagePath)
.input(pdfPath)
.exec(function (err, stdout, stderr) {
if (!err) {
console.log('gs executed w/o error')
console.log('stdout', stdout)
console.log('stderr', stderr)
resolve(imagePath)
} else {
console.log('gs error:', err)
reject(err)
}
})
} catch (error) {
console.log(error)
}
})
}
async function uploadImage (imagePath, appointment) {
const imagePathParts = imagePath.split('/')
const imageName = imagePathParts[imagePathParts.length - 1]
console.log(`Starting upload for ${imageName} at ${imagePath} to storage ${appointment}/${imageName}`)
await storage.bucket(GOOGLE_STORAGE_BUCKET_NAME).upload(imagePath, {
destination: `${appointment}/${imageName}`,
metadata: {
metadata: { appointment }
}
})
console.log(`Successfully uploaded image for appointment ${appointment}`)
}
function deleteDir (dir) {
rimraf.sync(dir)
}
Having an error might be a simple fix though when i console.log(${req.handle} & console.log(${req.email} & console.log(${req} they all return undefined?
below is my code, is there a reason it's not getting the req data?
exports.uploadImage = (req, res) => {
let db = firebase.firestore();
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs')
const busboy = new BusBoy ({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`feildname is ${fieldname}`);
console.log(`filename is ${filename}`);
console.log(`mimetype is ${mimetype}`);
console.log(`req email is ${req.email}`);
//image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//sfsgsdgsdgsdgsdgsdgsd.png
const imageFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = {filepath , mimetype };
file.pipe(fs.createWriteStream(filepath))
})
busboy.on('finish', () => {
//old syntax
///admin.storage().bucket().upload(imageToBeUploaded.filepath, {
admin.storage().bucket(`${config.storageBucket}`).upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl =
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageFileName}?alt=media`
console.log(`this is req = ${req.email}`)
return db.doc(`/users/${req.user.handle}`).update({ imageUrl});
// return db.doc(`/users/xxcell`).update({ imageUrl});
})
.then(() => {
return res.json({message: 'image uploaded successfully'})
})
.catch(err => {
console.error(err);
return res.status(500).json({error: err.code})
});
});
busboy.end(req.rawBody)
};
The request object in express exposes header parameters in the header function, like so:
app.get("/", (req, res) => {
console.log(`${req.header(email)}`)
res.send(200)
})