So I'm trying to programmatically delete Wasabi CDN objects from one of my buckets. My request is sending back 204 and showing success but nothing is being moved/deleted. I'm using node/javascript to do this.
Here is my function that is supposed to delete the bucket.
import expressAsyncHandler from 'express-async-handler'
import User from '../../models/User.js'
import axios from 'axios'
import aws4 from 'aws4'
/**
* #desc: THIS is going to be a testing function that will be added into admin delete user and all related docs.
* #route: DELETE
* #access: Private - Admin Route for when deleting user will delete the CDN username bucket aswell
* #goalHere: The goal of this function is to delete the user bucket from CDN. So that if we d
* #comment: This is a testing function that will be added into deleteVideo.js. Unless we just await this function in deleteVideo.js.
*/
export const deleteUserBucket = expressAsyncHandler(async (req, res, next) => {
try {
const user = await User.findOne({ username: req.user.username })
const username = user.username
let request = {
host: process.env.CDN_HOST,
method: 'DELETE',
url: `https://s3.wasabisys.com/truthcasting/${username}?force_delete=true`,
path: `/truthcasting/${username}?force_delete=true`,
headers: {
'Content-Type': 'application/json',
},
service: 's3',
region: 'us-east-1',
maxContentLength: Infinity,
maxBodyLength: Infinity,
}
let signedRequest = aws4.sign(request, {
accessKeyId: process.env.CDN_KEY,
secretAccessKey: process.env.CDN_SECRET,
})
//delete the Host and Content-Length headers
delete signedRequest.headers.Host
delete signedRequest.headers['Content-Length']
const response = await axios(signedRequest)
console.log(response.data)
console.log('successfully deleted user bucket', response.status)
return res.status(200).json({
message: `Successfully deleted user bucket`,
})
} catch (error) {
console.log(error)
return res.status(500).json({
message: `Problem with deleting user bucket`,
})
}
})
export default deleteUserBucket
When I send the http DELETE request in POSTMAN to {{dev}}api/admin/deleteuserbucket it then gives me a response of 204 ok and this is the response.
{
"message": "Successfully deleted user bucket"
}
I then go to my Wasabi CDN Buckets to check if it is deleted, in this case it's goodstock and it's still there. Feel like I'm missing something dumb here.
In S3, the delete bucket API call return 204 No content and an empty response body on successful delete.
With that URL, you are making a delete request on an object and not the bucket:
URL: `https://s3.wasabisys.com/truthcasting/${username}?force_delete=true`
The username passed in this URL will be interpreted as a key and S3 will look for an object in the root of the bucket.
Also why not using the AWS SDK to delete the bucket instead of reimplementing all of this. Check the AWS docs for this.
So for deleting contents that're inside of your root bucket, you need to point it to the complete object. That being said the way I had it set up in the original post code was returning 204("which is expected from Wasabi API") and not deleting anything due to the fact that I wasn't pointing it to the complete object path. Also I've found out that if you want to do batch delete instead of deleting one file, one by one you can use the aws-sdk node package to do a get request to your object, then use that response to loop through the object and remove what you need.. Here is an example. Hopefully this can help someone in the near future.
import expressAsyncHandler from 'express-async-handler'
import User from '../../models/User.js'
import axios from 'axios'
import aws4 from 'aws4'
import errorHandler from '../../middleware/error.js'
import AWS from 'aws-sdk'
/**
* #desc: THIS is going to be a testing function that will be added into admin delete user and all related docs.
* #route: DELETE
* #access: Private - Admin Route for when deleting user will delete the CDN username bucket aswell
* #goalHere: The goal of this function is to delete the user bucket from CDN. So that if we d
* #comment: This is a testing function that will be added into deleteVideo.js. Unless we just await this function in deleteVideo.js.
*/
export const deleteUserBucket = expressAsyncHandler(async (req, res, next) => {
const username = req.body.username
try {
// Connection
// This is how you can use the .aws credentials file to fetch the credentials
const credentials = new AWS.SharedIniFileCredentials({
profile: 'wasabi',
})
AWS.config.credentials = credentials
// This is a configuration to directly use a profile from aws credentials file.
AWS.config.credentials.accessKeyId = process.env.CDN_KEY
AWS.config.credentials.secretAccessKey = process.env.CDN_SECRET
// Set the AWS region. us-east-1 is default for IAM calls.
AWS.config.region = 'us-east-1'
// Set an endpoint.
const ep = new AWS.Endpoint('s3.wasabisys.com')
// Create an S3 client
const s3 = new AWS.S3({ endpoint: ep })
// The following example retrieves an object for an S3 bucket.
// set the details for the bucket and key
const object_get_params = {
Bucket: 'truthcasting',
Prefix: `${username}/`,
//Key: `cnfishead/videos/4:45:14-PM-5-6-2022-VIDDYOZE-Logo-Drop.mp4`,
// Key: `cnfishead/images/headshot.04f99695-photo.jpg`,
}
// get the object that we just uploaded.
// get the uploaded test_file
// s3.getObject(object_get_params, function (err, data) {
// if (err) console.log(err, err.stack) // an error occurred
// else console.log(data) // successful response
// })
// get the object that we just uploaded.
// get the uploaded test_file
await s3.listObjectsV2(object_get_params, (err, data) => {
if (err) {
console.log(err)
return res.status(500).json({
message: 'Error getting object',
error: err,
})
} else {
console.log(data)
//TODO Change this for loop to a async for loop. Like this: for await (const file of data.Contents) { }
for (let i = 0; i < data.Contents.length; i++) {
const object_delete_params = {
Bucket: 'truthcasting',
Key: data.Contents[i].Key,
}
s3.deleteObject(object_delete_params, (err, data) => {
if (err) {
console.log(err)
return res.status(500).json({
message: 'Error deleting object',
error: err,
})
} else {
console.log(data)
}
})
}
if (data.IsTruncated) {
console.log('Truncated')
getObjectFromBucket(req, res, next)
}
//console.log('Not Truncated')
res.status(200).json({
message: `Successfully retrieved + deleted ${data.Contents.length} objects`,
data: data,
})
}
})
} catch (error) {
console.log(error)
errorHandler(error, req, res)
}
})
export default deleteUserBucket
Related
The react-google-login from the client react app sends the response back to the Nodejs server with a post request-
client code -
import axios from 'axios';
import React, { Component } from 'react';
import GoogleLogin from 'react-google-login';
import refreshTokenSetup from '../../utils/refreshToken';
const clientId =
'xxxxxx-xfdgsdjg3gfxxxxxxxxxxx.apps.googleusercontent.com';
function Login() {
const onSuccess = (res) => {
console.log('Login Success: currentUser:', res.profileObj);
alert(
`Logged in successfully welcome ${res.profileObj.name} 😍. \n See console for full profile object.`
);
axios
.post('http://localhost:5000/auth/checkToken', { body: res.tokenId })
.then()
.catch((err) => {
console.log(err);
});
};
const onFailure = (res) => {
console.log('Login failed: res:', res);
alert(
`Failed to login. 😢 Please ping this to repo owner twitter.com/sivanesh_fiz`
);
};
return (
<div>
<GoogleLogin
clientId={clientId}
buttonText='Login'
onSuccess={onSuccess}
onFailure={onFailure}
cookiePolicy={'single_host_origin'}
style={{ marginTop: '100px' }}
isSignedIn={true}
/>
</div>
);
}
export default Login;
the backend route-
const { OAuth2Client } = require('google-auth-library');
const key = require('../config/key');
module.exports = {
checkToken: (req, res, next) => {
console.log('checking begins...', req.body);
const client = new OAuth2Client(key.GOOGLE_CLIENT_ID);
async function verify() {
const ticket = await client.verifyIdToken({
idToken: req.body,
audience: key.GOOGLE_CLIENT_ID, // Specify the CLIENT_ID of the app that accesses the backend
// Or, if multiple clients access the backend:
//[CLIENT_ID_1, CLIENT_ID_2, CLIENT_ID_3]
});
const payload = ticket.getPayload();
const userid = payload['sub'];
// If request specified a G Suite domain:
// const domain = payload['hd'];
}
verify().catch(console.error);
},
};
The above code is in reference to official Google Documentation available at- https://developers.google.com/identity/sign-in/web/backend-auth
Now everything works fine, user is signed in in the client side, the tokenId is sent back to the server and can be verified by console logging it, even on https://jwt.io/ but the following error is shown-
TypeError: jwt.split is not a function
at OAuth2Client.verifySignedJwtWithCertsAsync (E:\Projects\EAbackend\node_modules\google-auth-library\build\src\auth\oauth2client.js:528:30)
at OAuth2Client.verifyIdTokenAsync (E:\Projects\EAbackend\node_modules\google-auth-library\build\src\auth\oauth2client.js:394:34)
at processTicksAndRejections (internal/process/task_queues.js:93:5)
at async verify (E:\Projects\EAbackend\middleware\auth.js:9:22)
The main issue is that the example from Google doesn't really tell us what is expected as input to the verifyIdToken({options}) function.
This is what Google stated:
After Google returns an ID token, it's submitted by an HTTP POST method request, with the parameter name credential, to your login endpoint.
Which to me, is a little unclear of what is actually sent to the server to be verified. So here is a working example, copied/pasted from https://developers.google.com/identity/gsi/web/guides/verify-google-id-token, but with proper definitions for token and CLIENT_ID that are not mentioned on the Google site.
Server side Node JS:
exports.googleTokenChecker = (request, response) => {
const CLIENT_ID = request.body.clientId;
const token = request.body.credential;
// copied from Google example
const {OAuth2Client} = require('google-auth-library');
const client = new OAuth2Client(CLIENT_ID);
async function verify() {
const ticket = await client.verifyIdToken({
idToken: token,
audience: CLIENT_ID, // Specify the CLIENT_ID of the app that accesses the backend
// Or, if multiple clients access the backend:
//[CLIENT_ID_1, CLIENT_ID_2, CLIENT_ID_3]
});
const payload = ticket.getPayload();
const userid = payload['sub'];
// If request specified a G Suite domain:
// const domain = payload['hd'];
}
verify().catch(console.error);
}
Client side HTML to show what is sent to the backend:
<div id="g_id_onload"
data-client_id="CLIENT_ID.apps.googleusercontent.com"
data-callback="handleCredentialResponse"
data-auto_prompt="false">
</div>
<script>
function handleCredentialResponse(response) {
var xhr = new XMLHttpRequest();
xhr.open("POST", "http://localhost:3000/api/google_token_checker", true);
xhr.setRequestHeader('Content-Type', 'application/json');
xhr.send(JSON.stringify(response));
}
</script>
// contents of response parameter
// {
// clientId: 'CLIENT_ID.apps.googleusercontent.com',
// credential: 'JWT_HEADER.JWT_PAYLOAD.JWT_SIGNATURE',
// select_by: 'btn'
// }
Problem is in the idToken: req.body,
req.body has a body object in which the token was present, simply changing it to req.body.body solved the error.
The problem might be very begginner level but took a lot of my time and no online resourse was available which could point me in any direction.
Check the POST Request you will find the error.
I have been trying to upload files (mostly images) to firebase storage through firebase cloud function (onRequest method). I had to upload files from its base64 form. With the below code, i was able to achieve it, yet the file seems to be broken after upload.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
const bucket = admin.storage().bucket();
const database = admin.database();
const express = require('express');
const cors = require('cors');
const safetyLogsAPI = express();
safetyLogsAPI.use(cors({ origin: true }));
safetyLogsAPI.post('/', async (req, res) => {
try {
const {
attachments,
projectId
} = req.body;
const safetyKey = database.ref('safetyLogs')
.child(projectId)
.push().key;
if(attachments && Array.isArray(attachments)) {
if (attachments.length > 0) {
for (let index = 0; index < attachments.length; index++) {
const base64Obj = attachments[index];
const { content, fileName, contentType } = base64Obj;
const stream = require('stream');
const bufferStream = new stream.PassThrough();
bufferStream.end(Buffer.from(content, 'base64'));
const fullPath = `SafetyIncidentLog/Images/${projectId}/${safetyKey}/${fileName}`;
const file = bucket.file(fullPath);
const metadata = {
projectId,
safetyLogId: safetyKey,
createdTimestamp: Date.now(),
systemGenerated: 'false',
fileType: 'Image',
fileName,
path: fullPath
};
bufferStream.pipe(file.createWriteStream({
metadata: {
contentType,
metadata
},
public: true,
validation: "md5"
}))
.on('error', (err) => {
console.log('Error Occured');
console.log(err);
})
.on('finish', () => {
console.log('File Upload Successfull');
});
}
}
}
return res.status(200).send({
code: 200,
message: 'Success'
});
} catch (error) {
console.log(error);
return res.status(500).send({
code:500,
message: 'Internal Server Error',
error: error.message
});
}
});
module.exports = functions.https.onRequest(safetyLogsAPI);
I have tried this approach with both the prefix part data:image/png;base64 present and eliminated. In both ways i see broken image. So where have I gone wrong. Is there a better way to make it?
Thanks in advance.
Also, is the approach i try to do so is a recommended way?. For use cases like, profile picture upload, and conversation image attachments, is this way recommended, or the a direct upload from client is recommended?
With Cloud Functions HTTP triggers, the request is terminated and the function is shut down as soon as you send a respond to the client. Any asynchronous work that isn't finished might never finish.
What I'm seeing in your code is that you send a response before the upload is complete. I can see that your call to res.status(200).send() happens immediately after you start the upload. Instead, your code should wait to send the response until after it completes, perhaps using on('finish') and on('error').
I'm creating a feature that to retrieves GitHub user names via the GitHub API.
When sending the GET request from Postman, I got a server error that reads:
"Configuration property "githubClientId" is not defined"
Despite the fact that I called the config package with githubClientID defined in config/default.json, like this:
{
"mongoURI": "mongodb+srv://massas:oir#socialapp-2dg3r.mongodb.net/test?retryWrites=true&w=majority",
"jwtToken" : "oecret",
"githubClientId:": "ID",
"githubSecret": "SECRET"
}
// change the values for public viewing
Here's the code that's calling the API:
const express = require('express');
const request = require('request');
const config = require('config');
// #route GET api/profile/github/:username
// #desc Get user repos from username
// #acess Private
router.get('/github/:username', (req, res) => {
try {
const options = {
uri: `https://api.github.com/users/${
req.params.username
}/repos?per_page=5&sort=created:asc&client_id=${config.get(
'githubClientId'
)}&client_secret=${config.get('githubSecret')}`,
method: 'GET',
headers: {
'user-agent': 'node.js'
}
};
request(options, (error, response, body) => {
if (error) console.error(error);
if (response.statusCode !== 200) {
return res.status(404).json({
msg: 'No Github profile found'
});
}
res.json(JSON.parse(body));
});
} catch (err) {
console.error(err.message);
res.status(500).send('Server Error');
}
});
How can I resolve this error?
1- You need to change the options object from:
const options = {
uri: `https://api.github.com/users/${
req.params.username
}/repos?per_page=5&sort=created:asc&client_id=${config.get(
'githubClientId'
)}&client_secret=${config.get('githubSecret')}`,
method: 'GET',
headers: {
'user-agent': 'node.js'
}
};
to:
const uri = encodeURI(
`https://api.github.com/users/${req.params.username}/repos?per_page=5&sort=created:asc`
);
const headers = {
'user-agent': 'node.js',
Authorization: `token ${config.get('githubToken')}`
};
const gitHubResponse = await axios.get(uri, { headers });
2- Make sure to add a default.json file in config folder with your GitHub secret access token
{ "githubToken": "<yoursecrectaccesstoken>" }
3- Install axios in the root of the project, if you didn't install it
npm i axios
I'm using NestJS, Node, and Express for my backend and Angular for my frontend. I have a stepper where the user steps through and enters in information about themselves as well as a profile photo and any photos of their art that they want to post (it's a rough draft). I'm sending the files to the backend with this code:
<h2>Upload Some Photos</h2>
<label for="singleFile">Upload file</label>
<input id="singleFile" type="file" [fileUploadInputFor]= "fileUploadQueue"/>
<br>
<mat-file-upload-queue #fileUploadQueue
[fileAlias]="'file'"
[httpUrl]="'http://localhost:3000/profile/artPhotos'">
<mat-file-upload [file]="file" [id]="i" *ngFor="let file of fileUploadQueue.files; let i = index"></mat-file-upload>
</mat-file-upload-queue>
The front-end sends the photos as an array of files; I tried to change it so that it just sent a single file but could not get it working. I'm less focused on that because the user may need to upload multiple files, so I want to figure it out regardless. On the backend, I'm using multer, multer-s3, and AWS-SDK to help upload the files however it isn't working. Here is the controller code:
#Post('/artPhotos')
#UseInterceptors(FilesInterceptor('file'))
async uploadArtPhotos(#Req() req, #Res() res): Promise<void> {
req.file = req.files[0];
delete req.files;
// tslint:disable-next-line:no-console
console.log(req);
await this._profileService.fileupload(req, res);
}
Here is ProfileService:
import { Profile } from './profile.entity';
import { InjectRepository } from '#nestjs/typeorm';
import { Repository } from 'typeorm';
import { ProfileDto } from './dto/profile.dto';
import { Req, Res, Injectable, UploadedFile } from '#nestjs/common';
import * as multer from 'multer';
import * as AWS from 'aws-sdk';
import * as multerS3 from 'multer-s3';
const AWS_S3_BUCKET_NAME = 'blah';
const s3 = new AWS.S3();
AWS.config.update({
accessKeyId: 'blah',
secretAccessKey: 'blah',
});
#Injectable()
export class ProfileService {
constructor(#InjectRepository(Profile)
private readonly profileRepository: Repository<Profile> ){}
async createProfile( profileDto: ProfileDto ): Promise<void> {
await this.profileRepository.save(profileDto);
}
async fileupload(#Req() req, #Res() res): Promise<void> {
try {
this.upload(req, res, error => {
if (error) {
// tslint:disable-next-line:no-console
console.log(error);
return res.status(404).json(`Failed to upload image file: ${error}`);
}
// tslint:disable-next-line:no-console
console.log('error');
return res.status(201).json(req.file);
});
} catch (error) {
// tslint:disable-next-line:no-console
console.log(error);
return res.status(500).json(`Failed to upload image file: ${error}`);
}
}
upload = multer({
storage: multerS3({
// tslint:disable-next-line:object-literal-shorthand
s3: s3,
bucket: AWS_S3_BUCKET_NAME,
acl: 'public-read',
// tslint:disable-next-line:object-literal-shorthand
key: (req, file, cb) => {
cb(null, `${Date.now().toString()} - ${file.originalname}`);
},
}),
}).array('upload', 1);
}
I haven't implemented any middleware extending multer, but I don't think I have to. You can see in the controller I erase the files property on req and replace it with the file where it's value is just the first member of the files array but that was just to see if it would work if I send it something it was expecting, but it did not work then. Does anyone have any ideas regarding how I can fix this? Or can anyone at least point me in the right direction with a link to a relevant tutorial or something?
My first guess would be that you are using the FileInterceptor and multer. I assume FileInterceptor adds multer in the controller which makes it available to the #UploadedFile decorator. Which could cause a conflict to your later use of multer. Try removing the interceptor and see if that fixes the issue.
Also I am attaching how I am doing file uploads. I am only uploading single images and I am using the AWS SDK so I don't have to work with multer directly, but here is how I am doing it, it might be helpful.
In the controller:
#Post(':id/uploadImage')
#UseInterceptors(FileInterceptor('file'))
public uploadImage(#Param() params: any, #UploadedFile() file: any): Promise<Property> {
return this.propertyService.addImage(params.id, file);
}
Then my service
/**
* Returns a promise with the URL string.
*
* #param file
*/
public uploadImage(file: any, urlKey: string): Promise<string> {
const params = {
Body: file.buffer,
Bucket: this.AWS_S3_BUCKET_NAME,
Key: urlKey
};
return this.s3
.putObject(params)
.promise()
.then(
data => {
return urlKey;
},
err => {
return err;
}
);
}
Thanks Jedediah, I like how simple your code is. I copied your code however it still wasn't working. Turns out you have to instantiate the s3 object after you update the config with your accesskey and secretID.
I'm trying to figure out how to use ember-uploader, I have the following component (like the one in the README)
export default EmberUploader.FileField.extend({
filesDidChange: function(files) {
const uploader = EmberUploader.Uploader.create({
url: (ENV.APP.API_HOST || '') + '/api/v1/images/',
});
console.log(uploader);
if (!Ember.isEmpty(files)) {
var photo = files[0];
console.log(photo);
uploader.upload(photo)
.then(data => {
// Handle success
console.log("Success uploading file");
console.log(data);
}, error => {
// Handle failure
console.log("ERROR uploading file");
console.log(error);
});
}
}
});
The express API endpoint is listening for a POST request.
var saveImage = (req, res, next) => {
let body = req.body;
res.json({
data: body
});
};
But the body is empty after the request is done. I really don't know how to implement the API endpoint in order to get the file, I tried to see the req object and it doesn't contains the file.
Debugging it, After select a file using the component I get the following info in the console.
Seems that the API endpoint works because I get the following output:
POST /api/v1/images/ 200 27.284 ms - 11
But I can't get the file.
SOLUTION
In Express 4, req.files is no longer available on the req object by
default. To access uploaded files on the req.files object, use a
multipart-handling middleware like busboy, multer, formidable,
multiparty, connect-multiparty, or pez.
Following this blog, the code below was added to the API and the ember-uploader code posted in the question worked as expected.
import formidable from 'formidable';
var saveImage = (req, res, next) => {
var form = new formidable.IncomingForm();
form.parse(req);
form.on('fileBegin', function (name, file){
file.path = __dirname + '/tmp/' + file.name;
});
form.on('file', function (name, file){
res.json({
data: file.name
});
});
};