Error: Cloud Functions for Firebase spawn EACCES (ghostscript) - javascript

I tried using Firebase Cloud Functions to create a thumbnail of a PDF file.
After the call of gs I get the following error:
2018-06-12T11:29:08.685Z E makeThumbnail: Error: spawn EACCES
at exports._errnoException (util.js:1020:11)
at ChildProcess.spawn (internal/child_process.js:328:11)
at exports.spawn (child_process.js:370:9)
at Object.exec (/user_code/node_modules/gs/index.js:86:28)
at Promise (/user_code/index.js:95:12)
at mkdirp.then.then (/user_code/index.js:86:12)
2018-06-12T11:29:08.698166767Z D makeThumbnail: Function execution took 780 ms, finished with status: 'error'
Is it necessary to use a component like ghostscript to use a plan other than Spark?
In addition, my code. Maybe I just do not see my problem in the code
const functions = require('firebase-functions');
const mkdirp = require('mkdirp-promise');
const gcs = require('#google-cloud/storage')();
const admin = require('firebase-admin');
const spawn = require('child-process-promise').spawn;
const path = require('path');
const os = require('os');
const fs = require('fs');
const gs = require('gs');
const THUMB_MAX_HEIGHT = 200;
const THUMB_MAX_WIDTH = 200;
const THUMB_PREFIX = 'thumb_';
const gs_exec_path = path.join(__dirname, './lambda-ghostscript/bin/gs');
try{admin.initializeApp(functions.config().firebase); } catch(e) {}
exports.makeThumbnail = functions.storage.object().onFinalize((object) => {
const filePath = object.name;
const contentType = object.contentType;
const fileDir = path.dirname(filePath);
const fileName = path.basename(filePath);
const thumbFilePath = path.normalize(path.join(fileDir, `${THUMB_PREFIX} ${fileName}`));
const tempLocalFile = path.join(os.tmpdir(), filePath);
const tempLocalDir = path.dirname(tempLocalFile);
const tempLocalThumbFile = path.join(os.tmpdir(), thumbFilePath);
const tmp_dir = os.tmpdir();
if (fileName.startsWith(THUMB_PREFIX)) {
console.log('Is thumbnail');
return null;
}
const bucket = gcs.bucket(object.bucket);
const file = bucket.file(filePath);
const thumbFile = bucket.file(thumbFilePath);
const metadata = {
contentType: contentType,
};
return mkdirp(tmp_dir).then(() => {
console.log("Dir Created");
console.log(tempLocalFile);
return file.download({destination: tempLocalFile});
}).then(() => {
console.log("File downloaded");
if(!contentType.startsWith("image/")){
return new Promise((resolve, reject) => {
const pg= 1;
gs().batch().nopause()
.option(`-dFirstPage=${pg}`)
.option(`-dLastPage=${pg}`)
.executablePath(gs_exec_path)
.device('png16m')
.output(tempLocalThumbFile+".png")
.input(tempLocalFile)
.exec(err => err ? reject(err) : resolve());
});
}
else
{
var args = [ tempLocalFile, '-thumbnail', `${THUMB_MAX_WIDTH}x${THUMB_MAX_HEIGHT}>`, tempLocalThumbFile ];
return spawn('convert', args, {capture: ['stdout', 'stderr']});
}
}).then(() => {
return bucket.upload(tempLocalThumbFile, { destination: thumbFilePath });
}).then(() => {
fs.unlinkSync(tempLocalFile);
fs.unlinkSync(tempLocalThumbFile);
return result[0];
});
});

After hours of scratching my head and running same code over and over again pointlessly, I've finally found the problem!
The executable path that you've defined is not correct. It should be 'gs'.
Here's a complete gs() call sample:
gs()
.batch()
.option('-dFirstPage=2')
.option('-dLastPage=2')
.nopause()
.res(90)
.executablePath('gs')
.device('jpeg')
.output(tempNewPath2)
.input(tempFilePath)
.exec((err, stdout, stderr) => {
if (!err) {
console.log('gs executed w/o error');
console.log('stdout', stdout);
console.log('stderr', stderr);
resolve();
} else {
console.log('gs error:', err);
reject(err);
}
});
For more help, you can go through a sample repo that I created for this issue
https://github.com/krharsh17/ghostscript-firebase-sample

Related

how can QR async function code on a custom command in cypress

hi somebody can help me to create a custom command on cypress to use this async function
var jimp = require("jimp");
var qrCode = require('qrcode-reader');
var fs = require('fs')
var path = require("path")
const readQRCode = async (fileName) => {
const filePath = path.join(__dirname, fileName)
try {
if (fs.existsSync(filePath)) {
const img = await jimp.read(fs.readFileSync(filePath));
const qr = new qrCode();
const value = await new Promise((resolve, reject) => {
qr.callback = (err, v) => err != null ? reject(err) : resolve(v);
qr.decode(img.bitmap);
});
return value.result;
}
} catch (error) {
return error.message
}
}

Run code after executing promise in Javascript

I am trying to save to json the values returned from indeed api. I use indeed-scraper code from github https://github.com/rynobax/indeed-scraper
My code:
... required files ...
const parsedResults = []
indeed.query(queryOptions).then(response => {
response.forEach((res,i) => {
setTimeout(function(){
let url = res.url
let resultCount = 0
console.log(`\n Scraping of ${url} initiated...\n`)
const getWebsiteContent = async (url) => {
try {
const response = await axios.get(url)
const $ = cheerio.load(response.data)
...get scraped data...
parsedResults.push(metadata)
} catch (error) {
exportResults(parsedResults)
console.error(error)
}
}
getWebsiteContent(url)
}
, i*3000);
});
});
const outputFile = 'data.json'
const fs = require('fs');
const exportResults = (parsedResults) => {
fs.writeFile(outputFile, JSON.stringify(parsedResults, null, 4), (err) => {
if (err) {
console.log(err)
}
console.log(`\n ${parsedResults.length} Results exported successfully to ${outputFile}\n`)
})
}
parsedResults is not accessible in last portion of script, so to save as json file.
Any help appreciated!

Nodejs AWS Lambda s3 getObject method returns nothing

The script used when trying to get contents from the csv stored in the s3 bucket
const mysql = require("mysql");
const fs = require("fs");
const { google } = require("googleapis");
const AWS = require("aws-sdk");
const client = new AWS.SecretsManager({ region: "eu-west-1" });
const analyticsreporting = google.analyticsreporting("v4");
const csv = require('ya-csv')
const fastCsv = require('fast-csv')
const s3 = new AWS.S3();
const getParams = {
Bucket: 'data',
Key: 'athena_test/nameplate.csv'
};
exports.handler = async (context, event) => {
const data = await s3.getObject(getParams, function (err, data){
if(err){console.log("ERROR: ",err)}
else {return data}
})
console.log(data.Body)
}
the console log returns undefined rather than the contents of the csv
Hey you can try this one:-
const csv = require('#fast-csv/parse');
const s3Stream = await s3.getObject(params).createReadStream();
const data = await returnDataFromCSV();
console.log(data.Body);
const returnDataFromCSV =()=> {
let promiseData = new Promise((resolve, reject) => {
const parser = csv
.parseStream(csvFile, { headers: true })
.on("data", (data) => {
console.log('Parsed Data:-', data);
})
.on("end", ()=> {
resolve("CSV finished here");
})
.on("error",()=> {
reject("if failed");
});
});
try {
return await promiseData;
} catch (error) {
console.log("Get Error: ", error);
return error;
}
}
CreateStream: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Request.html#createReadStream-property

Convert image (jpg) to PDF via firebase function [duplicate]

Cloud Functions for Firebase has this nice sample where they create a thumbnail for each uploaded image. This is done by making use of ImageMagick.
I tried to convert the sample to convert PDFs to images. This is something ImageMagick can do, but I can't make it work with Cloud Functions for Firebase. I keep getting a code 1 error:
ChildProcessError: `convert /tmp/cd9d0278-16b2-42be-aa3d-45b5adf89332.pdf[0] -density 200 /tmp/cd9d0278-16b2-42be-aa3d-45b5adf89332.pdf` failed with code 1
at ChildProcess.<anonymous> (/user_code/node_modules/child-process-promise/lib/index.js:132:23)
at emitTwo (events.js:106:13)
at ChildProcess.emit (events.js:191:7)
at maybeClose (internal/child_process.js:877:16)
at Socket.<anonymous> (internal/child_process.js:334:11)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at Pipe._handle.close [as _onclose] (net.js:498:12)
Of course one possibility is that converting PDFs are simply not supported.
const functions = require('firebase-functions');
const gcs = require('#google-cloud/storage')();
const spawn = require('child-process-promise').spawn;
// [END import]
// [START generateThumbnail]
/**
* When an image is uploaded in the Storage bucket We generate a thumbnail automatically using
* ImageMagick.
*/
// [START generateThumbnailTrigger]
exports.generateThumbnail = functions.storage.object().onChange(event => {
// [END generateThumbnailTrigger]
// [START eventAttributes]
const object = event.data; // The Storage object.
const fileBucket = object.bucket; // The Storage bucket that contains the file.
const filePath = object.name; // File path in the bucket.
const contentType = object.contentType; // File content type.
const resourceState = object.resourceState; // The resourceState is 'exists' or 'not_exists' (for file/folder deletions).
// [END eventAttributes]
// [START stopConditions]
// Exit if this is triggered on a file that is not an image.
if (!contentType.startsWith('application/pdf')) {
console.log('This is not a pdf.');
return;
}
// Get the file name.
const fileName = filePath.split('/').pop();
// Exit if the image is already a thumbnail.
if (fileName.startsWith('thumb_')) {
console.log('Already a Thumbnail.');
return;
}
// Exit if this is a move or deletion event.
if (resourceState === 'not_exists') {
console.log('This is a deletion event.');
return;
}
// [END stopConditions]
// [START thumbnailGeneration]
// Download file from bucket.
const bucket = gcs.bucket(fileBucket);
const tempFilePath = `/tmp/${fileName}`;
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
console.log('Pdf downloaded locally to', tempFilePath);
// Generate a thumbnail of the first page using ImageMagick.
return spawn('convert', [tempFilePath+'[0]' ,'-density', '200', tempFilePath]).then(() => {
console.log('Thumbnail created at', tempFilePath);
// Convert pdf extension to png
const thumbFilePath = filePath.replace('.pdf', 'png');
// Uploading the thumbnail.
return bucket.upload(tempFilePath, {
destination: thumbFilePath
});
});
});
// [END thumbnailGeneration]
});
Node modules can install native code that is in the same directory as the Cloud Function's source code. I found that some node libraries on github that do this for ghostscript which is a very useful library for PDF processing:
Node library that wraps Ghostscript command line:
https://github.com/sina-masnadi/node-gs
Compiled Ghostscript which is
used via git submodule:
https://github.com/sina-masnadi/lambda-ghostscript
I put lambda-ghostscript into a sub-directory of my functions directory, then add the node-gs as a dependency in my package file like this:
{
"name": "functions",
"dependencies": {
"#google-cloud/storage": "^1.3.1",
"child-process-promise": "^2.2.1",
"firebase-admin": "~5.4.0",
"firebase-functions": "^0.7.2",
"gs": "https://github.com/sina-masnadi/node-gs/tarball/master"
}
}
Then in my index.js file I can just require the node library to easily use ghostscript from JavaScript. Here's the complete code for the Cloud Function that uses a Google Cloud Storage trigger:
const functions = require('firebase-functions');
const gcs = require('#google-cloud/storage')();
const spawn = require('child-process-promise').spawn;
const path = require('path');
const os = require('os');
const fs = require('fs');
var gs = require('gs');
exports.makePNG = functions.storage.object().onChange(event => {
// ignore delete events
if (event.data.resourceState == 'not_exists') return false;
const filePath = event.data.name;
const fileDir = path.dirname(filePath);
const fileName = path.basename(filePath);
const tempFilePath = path.join(os.tmpdir(), fileName);
if (fileName.endsWith('.png')) return false;
if (!fileName.endsWith('.pdf')) return false;
const newName = path.basename(filePath, '.pdf') + '.png';
const tempNewPath = path.join(os.tmpdir(), newName);
// // Download file from bucket.
const bucket = gcs.bucket(event.data.bucket);
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
console.log('Image downloaded locally to', tempFilePath);
return new Promise(function (resolve, reject) {
gs()
.batch()
.nopause()
.option('-r' + 50 * 2)
.option('-dDownScaleFactor=2')
.executablePath('lambda-ghostscript/bin/./gs')
.device('png16m')
.output(tempNewPath)
.input(tempFilePath)
.exec(function (err, stdout, stderr) {
if (!err) {
console.log('gs executed w/o error');
console.log('stdout',stdout);
console.log('stderr',stderr);
resolve();
} else {
console.log('gs error:', err);
reject(err);
}
});
});
}).then(() => {
console.log('PNG created at', tempNewPath);
// Uploading the thumbnail.
return bucket.upload(tempNewPath, {destination: newName});
// Once the thumbnail has been uploaded delete the local file to free up disk space.
}).then(() => {
fs.unlinkSync(tempNewPath);
fs.unlinkSync(tempFilePath);
}).catch((err) => {
console.log('exception:', err);
return err;
});
});
Here's the project on github: https://github.com/ultrasaurus/ghostscript-cloud-function
Disclaimer: This is using compiled native code and I verified experimentally that works for this case, so it is probably fine. I didn't look into the specific compile options and validate if they exactly correct for the Cloud Functions environment.
WORKING SOLUTION
Thank you #Ultrasaurus for pointing out this approach! However, for me it did not work and in your Github repo your also stated I haven't tested them. I modified your solution a little bit and got the following code, which is 100% working for me:
{
"dependencies": {
"#google-cloud/firestore": "^4.4.0",
"#google-cloud/storage": "^5.3.0",
"ghostscript": "https://github.com/musubu/node-ghostscript/tarball/master",
"pdf-image": "^2.0.0",
"rimraf": "^3.0.2",
"uuid": "^8.3.1"
}
}
The function is triggered by a Firestore event:
const Storage = require('#google-cloud/storage')
const fs = require('fs')
const rimraf = require('rimraf')
const os = require('os')
const gs = require('ghostscript')
const GOOGLE_PROJECT_ID = 'MY_GOOGLE_PROJECT_ID'
const GOOGLE_STORAGE_BUCKET_NAME = 'MY_GOOGLE_STORAGE_BUCKET_NAME'
const storage = new Storage.Storage({
projectId: GOOGLE_PROJECT_ID
})
exports.createImage = async (event) => {
let {
appointment,
name
} = event.value.fields
name = getFileName(name.stringValue)
appointment = appointment.stringValue
console.log(`Processing document ${name} in appointment ${appointment}`)
const tempDir = createTempDir(appointment)
const tmpDocumentPath = await downloadPdf(tempDir, name, appointment)
const imagePath = await convertPdfToImage(tmpDocumentPath)
await uploadImage(imagePath, appointment)
deleteDir(tempDir)
}
function getFileName (name) {
const nameParts = name.split('/')
return nameParts[nameParts.length - 1]
}
function createTempDir (appointment) {
const tempDir = `${os.tmpdir()}/${appointment}_${Math.random()}`
fs.mkdirSync(tempDir)
console.log(`Created dir ${tempDir}`)
return tempDir
}
async function downloadPdf (tempDir, name, appointment) {
const destination = `${tempDir}/${name}`
await storage.bucket(GOOGLE_STORAGE_BUCKET_NAME).file(`${appointment}/${name}`).download({ destination })
console.log(`Successfully downloaded document ${name}`)
return destination
}
async function convertPdfToImage (pdfPath) {
const imagePath = pdfPath.replace('pdf', 'png')
return new Promise(function (resolve, reject) {
try {
gs()
.batch()
.nopause()
.device('png16m')
.output(imagePath)
.input(pdfPath)
.exec(function (err, stdout, stderr) {
if (!err) {
console.log('gs executed w/o error')
console.log('stdout', stdout)
console.log('stderr', stderr)
resolve(imagePath)
} else {
console.log('gs error:', err)
reject(err)
}
})
} catch (error) {
console.log(error)
}
})
}
async function uploadImage (imagePath, appointment) {
const imagePathParts = imagePath.split('/')
const imageName = imagePathParts[imagePathParts.length - 1]
console.log(`Starting upload for ${imageName} at ${imagePath} to storage ${appointment}/${imageName}`)
await storage.bucket(GOOGLE_STORAGE_BUCKET_NAME).upload(imagePath, {
destination: `${appointment}/${imageName}`,
metadata: {
metadata: { appointment }
}
})
console.log(`Successfully uploaded image for appointment ${appointment}`)
}
function deleteDir (dir) {
rimraf.sync(dir)
}

Node.js TypeError: Cannot read property 'handle' of undefined when accessing req.handle

Having an error might be a simple fix though when i console.log(${req.handle} & console.log(${req.email} & console.log(${req} they all return undefined?
below is my code, is there a reason it's not getting the req data?
exports.uploadImage = (req, res) => {
let db = firebase.firestore();
const BusBoy = require('busboy');
const path = require('path');
const os = require('os');
const fs = require('fs')
const busboy = new BusBoy ({ headers: req.headers });
let imageFileName;
let imageToBeUploaded = {};
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
console.log(`feildname is ${fieldname}`);
console.log(`filename is ${filename}`);
console.log(`mimetype is ${mimetype}`);
console.log(`req email is ${req.email}`);
//image.png
const imageExtension = filename.split('.')[filename.split('.').length - 1];
//sfsgsdgsdgsdgsdgsdgsd.png
const imageFileName = `${Math.round(Math.random()*100000000000)}.${imageExtension}`;
const filepath = path.join(os.tmpdir(), imageFileName);
imageToBeUploaded = {filepath , mimetype };
file.pipe(fs.createWriteStream(filepath))
})
busboy.on('finish', () => {
//old syntax
///admin.storage().bucket().upload(imageToBeUploaded.filepath, {
admin.storage().bucket(`${config.storageBucket}`).upload(imageToBeUploaded.filepath, {
resumable: false,
metadata: {
metadata: {
contentType: imageToBeUploaded.mimetype
}
}
})
.then(() => {
const imageUrl =
`https://firebasestorage.googleapis.com/v0/b/${config.storageBucket}/o/${imageFileName}?alt=media`
console.log(`this is req = ${req.email}`)
return db.doc(`/users/${req.user.handle}`).update({ imageUrl});
// return db.doc(`/users/xxcell`).update({ imageUrl});
})
.then(() => {
return res.json({message: 'image uploaded successfully'})
})
.catch(err => {
console.error(err);
return res.status(500).json({error: err.code})
});
});
busboy.end(req.rawBody)
};
The request object in express exposes header parameters in the header function, like so:
app.get("/", (req, res) => {
console.log(`${req.header(email)}`)
res.send(200)
})

Categories