Google Cloud Storage (setting up) NodeJS - node.js

So I am looking at the sample code from Google and I can't work out how do I activate the config file?
https://cloud.google.com/appengine/docs/flexible/nodejs/using-cloud-storage
The sample code:
const {format} = require('util');
const express = require('express');
const Multer = require('multer');
const bodyParser = require('body-parser');
// By default, the client will authenticate using the service account file
// specified by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use
// the project specified by the GOOGLE_CLOUD_PROJECT environment variable. See
// https://github.com/GoogleCloudPlatform/google-cloud-node/blob/master/docs/authentication.md
// These environment variables are set automatically on Google App Engine
const {Storage} = require('#google-cloud/storage');
// Instantiate a storage client
const storage = new Storage();
const app = express();
app.set('view engine', 'pug');
app.use(bodyParser.json());
// Multer is required to process file uploads and make them available via
// req.files.
const multer = Multer({
storage: Multer.memoryStorage(),
limits: {
fileSize: 5 * 1024 * 1024, // no larger than 5mb, you can change as needed.
},
});
// A bucket is a container for objects (files).
const bucket = storage.bucket(process.env.GCLOUD_STORAGE_BUCKET);
// Display a form for uploading files.
app.get('/', (req, res) => {
res.render('form.pug');
});
// Process the file upload and upload to Google Cloud Storage.
app.post('/upload', multer.single('file'), (req, res, next) => {
if (!req.file) {
res.status(400).send('No file uploaded.');
return;
}
// Create a new blob in the bucket and upload the file data.
const blob = bucket.file(req.file.originalname);
const blobStream = blob.createWriteStream();
blobStream.on('error', (err) => {
next(err);
});
blobStream.on('finish', () => {
// The public URL can be used to directly access the file via HTTP.
const publicUrl = format(
`https://storage.googleapis.com/${bucket.name}/${blob.name}`
);
res.status(200).send(publicUrl);
});
blobStream.end(req.file.buffer);
});
const PORT = process.env.PORT || 8080;
app.listen(PORT, () => {
console.log(`App listening on port ${PORT}`);
console.log('Press Ctrl+C to quit.');
});
However when you read https://github.com/GoogleCloudPlatform/google-cloud-node/blob/master/docs/authentication.md it says to set up a config file and do the following
{
"projectId": "grape-spaceship-123",
"keyFilename": "./PROJECT-XXXXXX.json"
}
The keyFilename links to the google generated JSON.
But now how do I tell the sample code above to use that?
NOTE: Adding the config file
const config = require('./config')

I created storage and its work for me:
import { Storage } from '#google-cloud/storage';//may be you need to use require()
import * as path from 'path';
const storage = new Storage({
keyFilename: path.join(__dirname, '../********************.json'),
projectId: '***********Id'
})
const fileBucket = storage.bucket('***********-storage');
Good video about this: https://www.youtube.com/watch?v=pGSzMfKBV9Q

Related

Download File on Request in Firebase

I am looking for a solution to directly download a file in the Firebase Storage when hitting an API endpoint. I tried initializing a Google-Cloud Storage and downloading the file from the bucket.
const app = require('express')();
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({keyFilename: keyPath});
app.get("/download", (req, res) => {
storage.bucket(bucketName).file("file.txt").download({destination: './file.txt'});
});
app.listen(8080);
But this does not work!
I simply get:
UnhandledPromiseRejectionWarning: Error: Not Found
Could someone help me, please?
Where did you initialize the app
Original answer:
// Dependencies
const express = require('express')
const PORT = process.env.PORT || 3002;
// Initialize the App
const app = express();
// Start the app
app.listen(PORT, () => {
console.info(`Server is listening on port ${PORT}`);
});
Update:
Making HTTP requests to download files is an asynchronous operation. You need to wait for the file to be downloaded from the Google Cloud Storage before sending it to the client
const app = require('express')();
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({keyFilename: keyPath});
// I am using async/await here
app.get("/download", async (req, res) => {
// You have to wait till the file is downloaded
await storage.bucket(bucketName).file("file.txt").download({destination: './file.txt'});
// Send the file to the client
res.download('./file.txt')
});
app.listen(8080);
If the intention is to stream the file to the requesting client, you can pipe the data from Cloud Storage through to the response. It will look similar to the following:
const {Storage} = require('#google-cloud/storage');
const express = require('express');
const BUCKET_NAME = 'my-bucket';
const app = express();
const storage = new Storage({keyFilename: './path/to/service/key.json'});
app.get("/download", (req, res) => {
storage.bucket(bucketName).file("path/in/bucket/to/file.txt").createReadStream()
.on('error', (err) => {
res.status(500).send('Internal Server Error');
console.log(err);
})
.on('response', (storageResponse) => {
// make sure to check storageResponse.status
res.setHeader('content-type', storageResponse.headers['Content-Type']);
res.setHeader('content-length', storageResponse.headers['Content-Length']);
res.status(storageResponse.status);
// other headers may be necessary
// if status != 200, make sure to end the response as appropriate. (as it won't reach the below 'end' event)
})
.on('end', () => {
console.log('Piped file successfully.');
res.end();
}).pipe(res);
});
app.listen(8080);

Uploading multiple files to Google Cloud using multer and Node.js

I am trying to upload multiple files to Google Cloud bucket using Node.js and multer. It works with multer.single function but I don't know how to upload multiple images at once.
const bucket = gc.bucket('still-cover');
// Multer is required to process file uploads and make them available via
// req.files.
const multer = Multer({
storage: Multer.memoryStorage(),
limits: {
fileSize: 5 * 1024 * 1024, // no larger than 5mb, you can change as needed.
},
});
Router.post('/test/upload',multer.array('files',5),async(req,res)=>{
if (!req.files) {
res.status(400).send('No file uploaded.');
return;
}
// Create a new blob in the bucket and upload the file data.
const blob = bucket.file(req.files.originalname);
const blobStream = blob.createWriteStream();
blobStream.on('finish', res => {});
blobStream.on('finish', () => {
// The public URL can be used to directly access the file via HTTP.
const publicUrl = `https://storage.googleapis.com/${bucket.name}/${blob.name}`
res.status(200).send(publicUrl);
});
blobStream.end(req.files.buffer);
});
You can use multer.array('files', numberoffiles) or multer.any()to upload files to your Google Cloud Storage Bucket. You can use the following code to upload multiple files using Multer:
const express = require('express');
const path = require('path');
const cors = require('cors');
const Multer = require('multer');
const bodyParser = require('body-parser');
const {Storage} = require('#google-cloud/storage');
// Creates a client
const storage = new Storage();
const bucket = storage.bucket('YOUR_BUCKET_NAME')
const PATH = './public/';
const app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
const multer = Multer({
storage: Multer.memoryStorage(),
limits: {
fileSize: 5 * 1024 * 1024, // no larger than 5mb, you can change as needed.
},
});
app.get('/', function(req, res){
res.json({
json: "json"
});
})
// You can also use multer.array('data', numberofFiles)
app.post('/', multer.any(), function(req, res) {
console.log(req.files);
var counter = 0;
if (!req.files) {
res.status(400).send('No file uploaded.');
return;
}
// Create a new blob in the bucket and upload the file data.
req.files.forEach((fil) => {
const blob = bucket.file(fil.originalname);
const blobStream = blob.createWriteStream();
blobStream.on('finish', () => {
counter+=1
// The public URL can be used to directly access the file via HTTP.
const publicUrl = `https://storage.googleapis.com/${bucket.name}/${blob.name}`
if(counter>=2){
res.status(200).send(publicUrl);
}
});
blobStream.end(req.files.buffer);
});
});
app.listen(3000, function () {
console.log("Working on port 3000");
});
On the line blobStream.end(req.files.buffer); replace files.buffer with fil.buffer.

nodejs multer image upload : retain file name and extension

I have a node.js express code below to upload a image and store into a default folder.
I realised that the file gets renamed and the extension is lost. can some one help me fix this error?
1.How to retain extension and file name
if a zip file is upload, i want to unzip it and upload it
const __basefolder = "C:/Users/user/Desktop";
const express = require('express');
const multer = require('multer');
const upload = multer({dest: __basefolder + '/uploads/images'});
const app = express();
const PORT = 3000;
app.use(express.static('public'));
app.post('/upload', upload.single('file'), (req, res) => {
if(req.file) {
res.json(req.file);
}
else throw 'error';
});
app.listen(PORT, () => {
console.log('Listening at ' + PORT );
});
You can define a filename-option in your disk-storage setup, which lets you choose your filename arbitrarily. Since you want the original file-name, you can use file.originalname (note that using this an existing file with the same name will be overwritten though):
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, __basefolder + '/uploads/images');
},
filename: (req, file, cb) => {
cb(null, file.originalname);
}
})
const upload = multer({storage});
Regarding the second question: this has been answered before, see this or that for example.

S3 bucket: TypeError: Cannot read property 'transfer-encoding' of undefined

I am using first-time AWS' S3 bucket. I used node, express server, multer, and multerS3. For testing I used postman. I wanted to upload image to my s3 bucket. I have created the bucket also add my credentials to my backend. But when I am trying to upload an image by using postman, (this is how I did post request). I got error "TypeError: Cannot read property 'transfer-encoding' of undefined".
This is my s3 setup
const aws = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
aws.config.update({
secretAccessKey: "AKIAJWFJ6GS2*******",
accessKeyId: "W/2129vK2eLcwv67J******",
region: "us-east-1"
});
const s3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3: s3,
bucket: "testing-alak",
metadata: function(req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function(req, file, cb) {
cb(null, Date.now().toString());
}
})
});
module.exports = upload;
This is upload file setup
const express = require("express");
const router = express.Router();
const upload = require("./upload-file");
const singleUpload = upload.single("image");
router.post("/", (req, res) => {
singleUpload((req, res, next) => {
return res.json({
imgUrl: req.file.location
});
});
});
module.exports = router;
This is my express server
const express = require("express");
const app = express();
const route = require("./route");
const bodyParser = require("body-parser");
app.use(express.json()); //body Parser
app.use(bodyParser.urlencoded({ extended: true }));
app.use("/img", route);
const port = process.env.PORT || 5000;
app.listen(port, () => console.log(`🚀 App is listening at port ${port}!`));
If singleUpload is multer middleware, I've always used it like this:
router.post("/", singleUpload, (req, res) => {
return res.json({
imgUrl: req.file.location // this should be path?
});
});
Also, I don't think there is a location property. Maybe path is what you are looking for?
fieldname Field name specified in the form
originalname Name of the file on the user's computer
encoding Encoding type of the file
mimetype Mime type of the file
size Size of the file in bytes
destination The folder to which the file has been saved DiskStorage
filename The name of the file within the destination DiskStorage
path The full path to the uploaded file DiskStorage
buffer A Buffer of the entire file MemoryStorage

How use nodejs and postman to upload file into firebase?

I have a file nodejs that adds the picture(lion.jpg) to Cloud Storage
const firebase = require('firebase-admin');
const express = require('express');
const app = express();
const serviceAccount= require("./key9525")
firebase.initializeApp({
credential: firebase.credential.cert(serviceAccount),
databaseURL: "https://myadress.firebaseio.com" //example adress
});
const bucketName = 'myadress.appspot.com';
const filename ='./lion.jpg'; //example file
async function uploadFile() {
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
await storage.bucket(bucketName).upload(filename, {
gzip: true,
metadata: {
cacheControl: 'public, max-age=31536000',
},
});
console.log(`${filename} uploaded to ${bucketName}.`);
}
uploadFile();
and I have a file that allows me to select and upload a photo for example in postman
const express = require('express');
const app = express();
const port = 3000;
app.get('/', (req, res) => {
res.send('hello people');
});
app.listen(port, () => {
console.log('listening to the port: ' + port);
});
var multer = require('multer');
var upload = multer({dest:'uploads/'});
app.post('/single', upload.single('profile'), (req, res) => {
try {
res.send(req.file);
}catch(err) {
res.send(400);
}
}
How I can connect these codes so that after loading the nodejs file, then selecting the file in postman, the file was upload in firebase?
thank you for all tips
You have to pass the multer as the middleware to your POST call as like what you did. Please refer this link

Resources