We Keep Coding

Using "return" in Async Google Cloud Functions

I am somewhat new to coding and recently created this script in order to pull data from Zoom and push it to Google Drive via API. I am trying to push this to a Google Cloud Function, but when running it in a Cloud Function and console logging each step in the process, it seems like the uploadFile function, specifically the drive.files.create method, is being skipped. Every other step is being console logged, but neither the err or res is being logged after the drive.files.create method. Google Cloud Functions does not show errors, instead it shows OK and that the function took 1500ms to execute. It works fine on my local machine, I am only having issues in Cloud Functions. Any suggestions on how to get this to act right would be super helpful. Thank you!
const axios = require("axios");
require("dotenv").config();
const stream = require("stream");
const request = require("request");
const { google } = require("googleapis");
const KEYFILEPATH = "./credentials.json";
const SCOPES = ["https://www.googleapis.com/auth/drive"];
const auth = new google.auth.GoogleAuth({
keyFile: KEYFILEPATH,
scopes: SCOPES,
});
let today = new Date().toISOString();
let zoomAccessToken;
let zoomDownloadUrl;
///////////////////////////////////////////////////////////////// Searching for latest Town Hall recording in Google.
const searchFile = async (auth) => {
const service = google.drive({ version: "v3", auth });
const files = [];
try {
const res = await service.files.list({
corpora: "drive",
includeItemsFromAllDrives: true,
supportsAllDrives: true,
driveId: "XXXXXXXXXXXXXXXX",
q: '"XXXXXXXXXXXXXXX" in parents',
fields: "nextPageToken, files(id, name)",
spaces: "drive",
});
Array.prototype.push.apply(files, res.files);
const filesArray = res.data.files;
const filesName = filesArray.map((x) => x.name).sort().reverse()[0];
console.log(filesName);
return filesName;
} catch (err) {
throw err;
}
};
///////////////////////////////////////////////////////////////// Get Zoom OAuth access token.
const getAccessToken = async () => {
return axios({
method: "post",
url: `https://zoom.us/oauth/token?grant_type=account_credentials&account_id=${process.env.ZOOM_ACCOUNT_ID}`,
headers: {
Authorization: "Basic" +new Buffer.from(process.env.ZOOM_CLIENT_ID + ":" + process.env.ZOOM_CLIENT_SECRET).toString("base64"),
},
});
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's data.
const getRecordingData = async () => {
const token = await getAccessToken();
zoomAccessToken = await token.data.access_token;
return axios({
method: "get",
url: "https://api.zoom.us/v2/meetings/XXXXXXXXX/recordings",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${zoomAccessToken}`,
},
});
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's date.
const getRecordingDate = async () => {
const recording = await getRecordingData();
const lastRecordingDate = await recording.data.start_time;
const recordingDateFormatted = `${lastRecordingDate.substring(0,4)}.${lastRecordingDate.substring(5, 7)}.${lastRecordingDate.substring(8,10)} - Town Hall.mp4`;
return recordingDateFormatted;
};
///////////////////////////////////////////////////////////////// Get the latest Town Hall recording's download link.
const zoomDownloadLink = async () => {
const recording = await getRecordingData();
zoomDownloadUrl = `${recording.data.recording_files[0].download_url}?access_token=${zoomAccessToken}`;
return zoomDownloadUrl;
};
///////////////////////////////////////////////////////////////// Upload data from latest Town Hall recording's download link to Google Drive.
const uploadFile = async (auth) => {
const buffer = await zoomDownloadLink();
const bs = new stream.PassThrough();
request(buffer).pipe(bs);
const drive = google.drive({ version: "v3", auth });
var fileMetadata = {
name: `${today.substring(0, 4)}.${today.substring(5, 7)}.${today.substring(8,10)} - Town Hall.mp4`,
parents: ["XXXXXXXXXXXXXXXXX"],
};
var media = {
mimeType: "video/mp4",
body: bs,
};
drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
uploadType: "resumable",
supportsAllDrives: true,
},
function (err, res) {
if (err) {
console.log(err);
} else {
console.log("File Id: ", res.data.id);
}
}
);
};
///////////////////////////////////////////////////////////////// Compares Town Hall files in Google Drive and Zoom. If different, run uploadFile function.
exports.townHall = async () => {
const townHallFile = await searchFile(auth);
const lastRecordingDate = await getRecordingDate();
if (townHallFile != lastRecordingDate) {
await uploadFile(auth);
} else {
console.log("No Recording Today");
}
};

As you are calling an API inside a cloud function which is an async function but does not have a return statement, it will only execute the function but doesn't wait for the response, because the drive.files.create call is running.
So to fix that just need to await the result of the API. Just add
return await statement on the API call
like:
const uploadFile = async (auth) => {
const buffer = await zoomDownloadLink();
const bs = new stream.PassThrough();
request(buffer).pipe(bs);
const drive = google.drive({ version: "v3", auth });
var fileMetadata = {
name: `${today.substring(0, 4)}.${today.substring(5, 7)}.${today.substring(8,10)} - Town Hall.mp4`,
parents: ["XXXXXXXXXXXXXXXXX"],
};
var media = {
mimeType: "video/mp4",
body: bs,
};
return await drive.files.create(
{
resource: fileMetadata,
media: media,
fields: "id",
uploadType: "resumable",
supportsAllDrives: true,
},
function (err, res) {
if (err) {
console.log(err);
} else {
console.log("File Id: ", res.data.id);
}
}
);
};
Also, something important when you are calling APIs inside cloud functions is the time out. Check on your CF time out is enough to wait for the API call response.
Also, you can use the Promise function to force wait for the response:
const result = uploadFile(aut);
const _response = await Promise.all(result);

Firebase multiple image upload then call function

I am successfully batch uploading images to firebase, but for some reason, my data is not being uploaded afterwards.
I can confirm though, it does work perfectly ONLY on a fresh build for the first time? Super weird.
I am not sure what is going wrong as I am able to receive the urls fine after the fact.
Here is my function:
onUploadImages = () => {
let photo =
Platform.OS === 'ios'
? this.state.images.map(img => img.uri.replace('file://', ''))
: this.state.images.map(img => img.uri);
Promise.all(
photo.map((image, index) => {
const sessionId = new Date().getTime();
const Blob = RNFetchBlob.polyfill.Blob;
const fs = RNFetchBlob.fs;
window.XMLHttpRequest = RNFetchBlob.polyfill.XMLHttpRequest;
window.Blob = Blob;
let uploadBlob = null;
let mime = 'image/jpg';
const imageRef = firebase
.storage()
.ref('brandProducts/')
.child(`${this.props.userData.uid}`)
.child(`${sessionId}-${index}`);
return fs
.readFile(image, 'base64')
.then(data => {
return Blob.build(data, {
type: `${mime};BASE64`,
});
})
.then(blob => {
uploadBlob = blob;
return imageRef.put(blob, {contentType: mime});
})
.then(() => {
uploadBlob.close();
return imageRef.getDownloadURL();
});
}),
)
.then(results => {
//results is, here, [ urlFromFirst, urlFronSecond, ...]
const urls = {...this.state.urls};
results.forEach((r, i) => (urls[i] = r));
const postObj = {
...this.state.postObj,
urls,
};
//THIS WONT RUN (BUT I AM ABLE TO LOG PROPER postObj) here
return firebase
.firestore()
.collection('brandProducts')
.add(postObj);
})
.then(docRef => {
Actions.categories();
console.log('Document written with ID: ', docRef.id);
})
.catch(error => {
console.error(error);
});
};
Here is my postObj:
postObj: {
title: 'A Title',
price: 20,
description: 'Some text here',
webLink: 'a url as a string',
user: 'Username',
urls: {
'0': 'downloadUrl1',
'1': 'downloadUrl2',
'2': 'downloadUrl3',
}
},
I would appreciate any help i can get figuring out where this is going wrong. Cheers!

Firebase Firestore writes only working on the first attempt of a fresh build

I've been building an app with Firebase & React Native primarily using Firestore. I started to use Firestore and its been great, but for some reason when writing to Firestore, it is only working on the first attempt (when i remove the app, rebuild, and perform my write).
I tried to do the exact same thing except write to Firestore and everything works as expected.
I am also receiving no error!
Here is what I am doing:
export const addBrandProduct = (postObj) => {
return () => {
firebase
.firestore()
.collection('brandProducts')
.add(postObj)
.then((docRef) => {
console.log("Document written with ID: ", docRef.id);
Actions.categories();
})
.catch(error => {
console.error("Error adding document: ", error);
});
};
};
For more of a reference, here is my component code that calls addBrandProduct()
onUploadImages = () => {
let photo =
Platform.OS === 'ios'
? this.state.images.map(img => img.uri.replace('file://', ''))
: this.state.images.map(img => img.uri);
photo.forEach((image, i) => {
const sessionId = new Date().getTime();
const Blob = RNFetchBlob.polyfill.Blob;
const fs = RNFetchBlob.fs;
window.XMLHttpRequest = RNFetchBlob.polyfill.XMLHttpRequest;
window.Blob = Blob;
let uploadBlob = null;
let mime = 'image/jpg';
const imageRef = firebase
.storage()
.ref('brandProducts/')
.child(`${this.props.userData.uid}`)
.child(`${sessionId}-${i}`);
fs.readFile(image, 'base64')
.then(data => {
return Blob.build(data, {type: `${mime};BASE64`});
})
.then(blob => {
uploadBlob = blob;
return imageRef.put(blob, {contentType: mime});
})
.then(() => {
uploadBlob.close();
return imageRef.getDownloadURL();
})
.then(url => {
//if this is the last uploaded image, post data to db
if (i === this.state.images.length - 1) {
const urls = {
...this.state.urls,
[i]: url,
};
const postObj = {
...this.state.postObj,
urls: urls,
};
this.props.addBrandProduct(postObj);
} else {
this.setState({
urls: {
...this.state.urls,
[i]: url,
},
});
}
})
.catch(error => {
console.log(error);
});
});
};
Basically, I am uploading a maximum of 3 images along with some data for it. In order to ensure I am uploading them all prior to adding the post data (writing to firestore) I am using a forEach and on the last upload, when it completes, I am calling the action to write the post data.

Edition
Hum addBrandProduct is a function that create another function.
So when you call this.props.addBrandProduct(postObj) nothing is sent to firestore, you just create a new function that should be called.
Maybe you can go out this stuff and call firebase directly, ensuring that everything works and then go back to the redux way if you still want to use it. I also make it parallelized instead of sequentials. Hope it help, hard to find the real problem when it can come from anywhere.
onUploadImages = () => {
let photo = Platform.OS === 'ios'
? this.state.images.map(img => img.uri.replace('file://', ''))
: this.state.images.map(img => img.uri);
Promise.all( photo.map( image => {
const sessionId = new Date().getTime();
const Blob = RNFetchBlob.polyfill.Blob;
//This is kind useless
//const fs = RNFetchBlob.fs;
//This is not used
//window.XMLHttpRequest = RNFetchBlob.polyfill.XMLHttpRequest;
//This is not adviced
//window.Blob = Blob;
let uploadBlob = null;
let mime = 'image/jpg';
const imageRef = firebase
.storage()
.ref('brandProducts/')
.child(`${this.props.userData.uid}`)
.child(`${sessionId}-${i}`);
return fs.readFile(image, 'base64')
.then(data => {
return RNFetchBlob.polyfill.Blob.build(data, {type: `${mime};BASE64`});
})
.then(blob => {
uploadBlob = blob;
return imageRef.put(blob, {contentType: mime});
})
.then(() => {
uploadBlob.close();
return imageRef.getDownloadURL();
});
))
.then( results => {
//results is, here, [ urlFromFirst, urlFronSecond, ...]
const urls = { ...this.state.urls};
results.forEach( (r, i) => urls[i] = r );
const postObj = {
...this.state.postObj,
urls
};
return firebase
.firestore()
.collection('brandProducts')
.add(postObj)
})
.then( docRef => {
console.log("Document written with ID: ", docRef.id);
})
.catch(error => {
console.error(error);
});
};

Upload Image from form-data to S3 using a Lambda

So I am writing a Lambda that will take in some form data via a straight POST through API Gateway (testing using Postman for now) and then send that image to S3 for storage. Every time I run it, the image uploaded to S3 is corrupted and won't open properly. I have seen people having to decode/encode the incoming data but I feel like I have tried everything using Buffer.from. I am only looking to store either .png or .jpg. The below code does not reflect my attempts using Base64 encoding/decoding seeing they all failed. Here is what I have so far -
Sample Request in postman
{
image: (uploaded .jpg/.png),
metadata: {tag: 'iPhone'}
}
Lambda
const AWS = require('aws-sdk')
const multipart = require('aws-lambda-multipart-parser')
const s3 = new AWS.S3();
exports.handler = async (event) => {
const form = multipart.parse(event, false)
const s3_response = await upload_s3(form)
return {
statusCode: '200',
body: JSON.stringify({ data: data })
}
};
const upload_s3 = async (form) => {
const uniqueId = Math.random().toString(36).substr(2, 9);
const key = `${uniqueId}_${form.image.filename}`
const request = {
Bucket: 'bucket-name',
Key: key,
Body: form.image.content,
ContentType: form.image.contentType,
}
try {
const data = await s3.putObject(request).promise()
return data
} catch (e) {
console.log('Error uploading to S3: ', e)
return e
}
}
EDIT:
I am now atempting to save the image into the /tmp directory then use a read stream to upload to s3. Here is some code for that
s3 upload function
const AWS = require('aws-sdk')
const fs = require('fs')
const s3 = new AWS.S3()
module.exports = {
upload: (file) => {
return new Promise((resolve, reject) => {
const key = `${Date.now()}.${file.extension}`
const bodyStream = fs.createReadStream(file.path)
const params = {
Bucket: process.env.S3_BucketName,
Key: key,
Body: bodyStream,
ContentType: file.type
}
s3.upload(params, (err, data) => {
if (err) {
return reject(err)
}
return resolve(data)
}
)
})
}
}
form parser function
const busboy = require('busboy')
module.exports = {
parse: (req, temp) => {
const ctype = req.headers['Content-Type'] || req.headers['content-type']
let parsed_file = {}
return new Promise((resolve) => {
try {
const bb = new busboy({
headers: { 'content-type': ctype },
limits: {
fileSize: 31457280,
files: 1,
}
})
bb.on('file', function (fieldname, file, filename, encoding, mimetype) {
const stream = temp.createWriteStream()
const ext = filename.split('.')[1]
console.log('parser -- ext ', ext)
parsed_file = { name: filename, path: stream.path, f: file, type: mimetype, extension: ext }
file.pipe(stream)
}).on('finish', () => {
resolve(parsed_file)
}).on('error', err => {
console.err(err)
resolve({ err: 'Form data is invalid: parsing error' })
})
if (req.end) {
req.pipe(bb)
} else {
bb.write(req.body, req.isBase64Encoded ? 'base64' : 'binary')
}
return bb.end()
} catch (e) {
console.error(e)
return resolve({ err: 'Form data is invalid: parsing error' })
}
})
}
}
handler
const form_parser = require('./form-parser').parse
const s3_upload = require('./s3-upload').upload
const temp = require('temp')
exports.handler = async (event, context) => {
temp.track()
const parsed_file = await form_parser(event, temp)
console.log('index -- parsed form', parsed_file)
const result = await s3_upload(parsed_file)
console.log('index -- s3 result', result)
temp.cleanup()
return {
statusCode: '200',
body: JSON.stringify(result)
}
}
The above edited code is a combination of other code and a github repo I found that is trying to achieve the same results. Even with this solution the file is still corrupted

Figured out this issue. Code works perfectly fine - it was an issue with API Gateway. Need to go into the API Gateway settings and set thee Binary Media Type to multipart/form-data then re-deploy the API. Hope this helps someone else who is banging their head against the wall on figuring out sending images via form data to a lambda.

Relation between storage and database - Firebase

I'm facing yet another issue.
I'm using firebase db to store text and firebase storage to store files. And here comes my issue.
Q: How to fetch a correct image from storage when fetching particular element from database?
Here's my attempt:
const storageRef = firebase.storage().ref('companyImages/companyImage' + 123);
^^^^^^^^^^^^ I dont have access to id yet :(
const task = storageRef.put(companyImage);
task.on('state_changed', () => {
const percentage = (snap.bytesTransferred / snap.totalBytes) * 100;
// ^^^^^^^^^^^^ not sure if i even need this
}, (err) => {
console.log(err);
}, () => {
firebase.database().ref('offers').push(values);
^^^^^^^^^^^^^^^ now I could retrieve id from it with .key but its too late
});
As you can see, first what Im doing is uploading the image and when it's succesful, Im starting to upload the data to database.
Still, it doesnt work as it is supposed to. When uploading image I have to name it with a correct id to retrieve it easily later, in components.
It may look a lil bit complex but will appreciate any kind of help. Any suggestion or hint.
Should I firstly upload data to DB and then image to the storage?

You can generate the push ID before you upload the file – you can also just save the download URL of the returned snapshot at task.snapshot.downloadURL so you don't have to retrieve the file from storage using the storage ref.
const offerRef = firebase.database().ref('offers').push();
const storageRef = firebase.storage().ref(`companyImages/${offerRef.key}`);
const task = storageRef.put(companyImage);
task.on('state_changed', (snap) => {
const percentage = (snap.bytesTransferred / snap.totalBytes) * 100;
}, (error) => {
console.log(err);
}, () => {
offerRef.set(values);
});

I would suggest using .getDownloadURL(). Then push all your uploadedfileDownloadURL's into an object or array and then store that into your database. So in the future you can access this object or array from, lets say your user/ProfilePHotos, and then in your app level code you can just use the DownloadURL as a uri links inside an image tag!
In this example I am using react-native, I upload multiple photos, save the download URL each time in an array, then set the array to firebase under the users account.
export const userVehiclePhotoUploadRequest = (photos, user, year) => dispatch => {
console.log('Inside vehiclePhotoUpload Actions', photos, user)
let referenceToUploadedPhotos = [];
return new Promise((resolve, reject) => {
photos.map(ele => {
let mime = 'application/octet-stream'
let uri = ele.uri
let uploadUri = Platform.OS === 'ios' ? uri.replace('file://', '') : uri
let sessionId = new Date().getTime()
let uploadBlob = null
let imageRef = firebase.storage().ref('vehicleImages/' + `${user.account.uid}`).child(`${sessionId}`)
fs.readFile(uploadUri, 'base64')
.then((data) => {
return Blob.build(data, { type: `${mime};BASE64` })
})
.then((blob) => {
uploadBlob = blob
return imageRef.put(blob, { contentType: mime })
})
.then(() => {
uploadBlob.close()
return imageRef.getDownloadURL()
})
.then((url) => {
referenceToUploadedPhotos.push(url)
console.log('ARRAY OF URLS WHILE PUSHING', referenceToUploadedPhotos)
resolve(url)
})
.catch((error) => {
reject(error)
})
})
})
.then(() => {
//I did this to not go home until photos are done uploading.
let vehicles;
firebase.database().ref('users/' + user.account.uid + `/allVehicles/allVehiclesArray`).limitToFirst(1).once('value').then(function (snapshot) {
// ******** This method is straight from their docs ********
// ******** It returns whatever is found at the path xxxxx/users/user.uid ********
vehicles = snapshot.val();
}).then(() => {
console.log('ARRAY OF URLS BEFORE SETTING', referenceToUploadedPhotos)
// let lastVehicle = vehicles.length - 1;
firebase.database().ref('users/' + user.account.uid + `/allVehicles/allVehiclesArray/` + `${Object.keys(vehicles)[0]}` + `/photosReference`).set({
referenceToUploadedPhotos
}).then(() => {
dispatch(loginRequest(user.account))
})
})
})
};
And then in your code, lets say inside a map of the user's information...
{ ele.photosReference !== undefined ? dynamicAvatar = { uri: `${ele.photosReference.referenceToUploadedPhotos[0]}` } : undefined }