Get readable stream from drive.files.get - node.js

I am writing a nodejs function to transfer files from google drive to some other place
Following is the sample code block to get file from drive
var fileId = '0BwwA4oUTeiV1UVNwOHItT0xfa2M';
var dest = fs.createWriteStream('/tmp/photo.jpg');
drive.files.get({
fileId: fileId,
alt: 'media'
})
.on('end', function () {
console.log('Done');
})
.on('error', function (err) {
console.log('Error during download', err);
})
.pipe(dest);
instead of piping it to writable stream , I would like to get a readable stream that i can pass on to my function
uploadsomeplace(readableStream)
Update 1
i wrote a function to get that readable stream and upload it again to some folder in google drive (seems dumb but for test as drive.files.creates takes a readable stream)
const transferit = async () => {
var fileId = '0BwwA4oUTe1UVNwOHItT0xfa2M' // some valid id
let somereadstream = drive.files.get(
{ fileId, alt: 'media' },
{ responseType: 'stream' }
)
var folderId = '1oC90HZDLbfYiW2neSCMERHZ8ZH1X' // some valid id
var fileMetadata = {
name: 'photo.jpg',
parents: [folderId]
}
var media = {
mimeType: 'image/jpeg',
// body: fs.createReadStream('image.jpg')
body: somereadstream
}
return await drive.files.create({
resource: fileMetadata,
media: media
// fields: 'id'
})
}
transferit().then(data => console.log(data))
but i get error
(node:6200) UnhandledPromiseRejectionWarning: TypeError: part.body.pipe is not a function

drive.files.get returns a promise which if fulfilled gives a response object which contains the readable stream in response.data
const isStream = require('is-stream')
const transferit = async () => {
var fileId = '1uw5PhBF8z8uBsxmFKJef582Pp92UKpxM' // some valid id
let somereadstream = await drive.files.get(
{ fileId, alt: 'media' },
{ responseType: 'stream' }
)
console.log(somereadstream)
return isStream.readable(somereadstream.data)
}

By the looks of things
drive.files.get({
fileId: fileId,
alt: 'media'
})
already returns a readable stream. Just assign it to a variable and pass it into your function.
const readableStream = drive.files.get({
fileId: fileId,
alt: 'media'
});
uploadsomeplace(readableStream);

This works for me in typescript:
async function getReadableStreamForFileId(
drive: drive_v3.Drive,
fileId: FileId,
): Promise<stream.Readable> {
console.log('getReadableStreamForFileId', {drive, fileId});
// https://developers.google.com/drive/api/v3/manage-downloads#node.js
return (await drive.files.get(
{fileId, alt: 'media'},
{responseType: 'stream'},
)).data as any as Promise<stream.Readable>;
}
example:
const fileId = '1F2*******************xGC';
const fileDataStream = await getReadableStreamForFileId(drive, fileId);
const target = fs.createWriteStream('data-out');
fileDataStream.pipe(target);
drive is initialized as described in https://developers.google.com/drive/api/v3/quickstart/nodejs

Related

Upload image to s3 bucket - react native and node js

Within my app a user can select a profile image and i would like that image to be uploaded to an s3 bucket when the user saves their profile data
I pass the image data (and json, which consists of name, email, telephone for example) from my app to an express server and upload there
At present I can pass the image data (the url it seems at present) to an s3 bucket and it saves
I don't think i'm actually saving the image itself though, as when downloading from s3 (manually) and trying to open on my mac it states it may be damaged and i cannot see the image
Feel daft for asking but how do i actually upload the image itself? Thanks
React Native Side
const handleFormSubmit = formData => {
const jsonData = JSON.stringify({
...formData,
});
// Handle profile image
if (imageProps && imageProps.uri) {
const data = new FormData();
data.append('formBody', jsonData);
data.append('image', {
uri:
Platform.OS === 'android'
? imageProps.uri
: imageProps.uri.replace('file://', ''),
type: imageProps.type,
name: imageProps.fileName,
});
sendRequest(data);
} else {
sendRequest(jsonData);
}
};
const sendRequest = data => {
let responseData;
fetch('http://localhost:8080/users/api/update_user_profile', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
body: data,
})
.then(response => {
responseData = response;
return response.json();
})
.then(jsonData => {
console.log(jsonData)
})
.catch(error => {
console.log(error)
});
};
Server Side
const s3 = new AWS.S3({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
});
// Setting up S3 upload parameters
const params = {
Bucket: 'bucket-folder',
ACL: 'public-read',
Key: req.files.image.name,
Body: req.files.image.path
};
const stored = await s3.upload(params).promise();
You can use Multer for uploading files to s3.
const multer = require('multer');
const AWS = require('aws-sdk');
const uniqid = require('uniqid');
const storage = multer.memoryStorage();
const upload = multer({ storage });
// ? Posts new file to amazon and saves to db
router.post(
'/:id',
upload.single('attachment'),
async (req, res) => {
const unique = uniqid.time();
const { file } = req;
const { filePath } = req.body;
const { id } = req.params;
const s3FileURL = process.env.AWS_UPLOADED_FILE_URL;
const region = process.env.AWS_REGION;
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
const Bucket = process.env.AWS_BUCKET_NAME + '/' + filePath;
const Key = `${id}/${unique}-${file.originalname}`;
const Body = file.buffer;
const ContentType = file.mimetype;
const ACL = 'public-read';
const s3bucket = new AWS.S3({
accessKeyId,
secretAccessKey,
region,
});
const params = {
Bucket,
Key,
Body,
ContentType,
ACL,
};
s3bucket.upload(params, async (err, data) => {
if (err) {
res.status(500).json({ error: true, Message: err });
} else {
console.log(params);
const newFileUploaded = {
description: req.body.description,
fileLink: `${s3FileURL}${filePath}/${id}/${unique}-${file.originalname}`,
s3_key: params.Key,
};
try {
const response = await postFile({
name: req.body.name,
attachment: newFileUploaded,
alt: req.body.alt,
user: req.body.user,
relatedID: req.body.relatedID,
});
res.status(200).json({
message: response.message,
success: response.success,
result: response.result,
});
} catch (e) {
res.status(500).json({
message:
'File upoladed but Db couldnt saved request (upload by ID)',
success: false,
result: [],
});
}
}
});
}
);

How to upload a file in a folder GOOGLE DRIVE API

So i try this, i use my folder ID, but that's doesn't work, my files are create but they don't go in my folder. In the doc of Google Drive there is only this example. When i submit a form on Slack, that's push my data on a Google Spreadsheet and after that's auto generate a ticket on monday. Now i want to push my spreadsheet on a special folder in my Drive.. Can i provide my files in an other way ?
async function createSpreadsheet(filename, body) {
const ID_FOLDER_DRIVE = MY_ID;
try {
const fileMetaData = {
'name': filename,
parents: [ID_FOLDER_DRIVE]
}
const media = {
mimeType: 'text/csv',
body: body
}
const res = await drive.files.create({
requestBody: {
resource: fileMetaData,
mimeType: 'application/vnd.google-apps.spreadsheet'
},
media: media
})
return res.data
} catch (error) {
throw error
}
}
async function generatePublicUrl(id) {
console.log({ id })
try {
await drive.permissions.create({
fileId: id,
requestBody: {
role: 'reader',
type: 'anyone',
},
});
/*
webViewLink: View the file in browser
webContentLink: Direct download link
*/
const result = await drive.files.get({
fileId: id,
fields: 'webViewLink, webContentLink',
});
return result.data.webViewLink
} catch (error) {
throw error
}
}
In your situation, how about the following modification?
From:
const fileMetaData = {
'name': filename,
parents: [ID_FOLDER_DRIVE]
}
const media = {
mimeType: 'text/csv',
body: body
}
const res = await drive.files.create({
requestBody: {
resource: fileMetaData,
mimeType: 'application/vnd.google-apps.spreadsheet'
},
media: media
})
To:
const fileMetaData = {
name: filename,
parents: [ID_FOLDER_DRIVE],
mimeType: 'application/vnd.google-apps.spreadsheet'
}
const media = {
mimeType: 'text/csv',
body: body
}
const res = await drive.files.create({
requestBody: fileMetaData,
media: media
})
Reference:
google-api-nodejs-client

nodejs google drive api file download error with readablestream object

I'm trying to implement a function for downloading a file from the google drive using google drive api V3.
Below is my code.
service.files.get({
auth: this.oauth2Client,
fileId: fileId,
alt: 'media'
}, {responseType: "stream"}, function (err, response) {
console.log(response);
response.data
.on('end',() => {
console.log('Done');
})
.on('error', (err) => {
console.log('Error during download', err);
})
.on('data', d=> {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write('Download ${progress} bytes');
}
})
.pipe(dest);
})
When I run the code, I get this error.
response.data.on is not a function
Not just the on function, but the pipe function doesn't work either.
I already checked that response.data is the object of Readablestream.
Any ideas?
UPDATES
Below is the full code for GoogleDriveSyn class.
The getAllFilesInFolder function works fine.
const path = require('path')
const {google} = require('googleapis');
const fs = require('fs');
const service = google.drive('v3');
class GoogleDriveSyn {
constructor(auth) {
var apiKeys = JSON.parse(fs.readFileSync('config.json'));
this.oauth2Client = new google.auth.OAuth2(
apiKeys['CLIENT_ID'],
apiKeys['SECRET_ID'],
"http://localhost:8080"
);
this.oauth2Client.credentials = auth;
this.directory = localStorage.getItem('directory');
}
getAllFilesInFolder(folderId) {
var query = '"' + folderId + '"' + ' in parents and mimeType!="application/vnd.google-apps.folder"';
service.files.list({
auth: this.oauth2Client,
q: query,
pageSize: 50,
fields: 'nextPageToken, files(id, name, modifiedTime, kind, createdTime, thumbnailLink, mimeType, size, webContentLink)'
}, function(err, response) {
if (err) {
console.log('The API returned an error: ' + err);
return;
}
console.log(response);
return response
})
}
downloadFile(fileId, filePath, fileName) {
var fullPath = path.join(this.directory, filePath, fileName);
var dest = fs.createWriteStream(fullPath);
let progress = 0;
service.files.get({
auth: this.oauth2Client,
fileId: fileId,
alt: 'media'
}, {responseType: "stream"}, function (err, response) {
console.log(response);
response.data
.on('end',() => {
console.log('Done');
})
.on('error', (err) => {
console.log('Error during download', err);
})
.on('data', d=> {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write('Download ${progress} bytes');
}
})
.pipe(dest);
})
}
}
module.exports = GoogleDriveSyn;
Also, below is the log of response inside downloadFile function.
{config: {…}, data: ReadableStream, headers: {…}, status: 200, statusText: ""}
config: {url: "https://www.googleapis.com/drive/v3/files/1zcxy0wWsPuWINMY8FP_bjR4nrnj6t8eD?alt=media", method: "GET", responseType: "stream", headers: {…}, paramsSerializer: ƒ, …}
data: ReadableStream {locked: false}
headers: {alt-svc: "h3-29=":443"; ma=2592000,h3-27=":443"; ma=2592000,…3"; ma=2592000,quic=":443"; ma=2592000; v="46,43"", cache-control: "private, max-age=0, must-revalidate", content-disposition: "attachment", content-length: "12", content-type: "text/plain", …}
status: 200
statusText: ""
__proto__: Object
I believe your goal and situation as follows.
You want to download a text file from the Google Drive.
Your script for authorizing works fine. So this.oauth2Client can be used for downloading the file.
Modification points:
In my environment, I confirmed that your script worked. But in your question, you say the error of response.data.on is not a function occurs. So in this modification, downloadFile was modified as follows.
At first, please confirm the version of googleapis for Node.js you are using. In this case, please use the latest version of googleapis for Node.js. In the current stage, the latest version is googleapis#55.0.0.
If you use the old one, I thought that the reason of your issue might be this.
In your script, 'Download ${progress} bytes' is not correct. In this case, please use it as the template literal. By this, the progress information can be seen.
Modified script:
downloadFile(fileId, filePath, fileName) {
var fullPath = path.join(this.directory, filePath, fileName);
var dest = fs.createWriteStream(fullPath);
let progress = 0;
return drive.files
.get(
{ auth: this.oauth2Client, fileId: fileId, alt: "media" },
{ responseType: "stream" }
)
.then((response) => {
return new Promise((resolve, reject) => {
response.data
.on("end", () => {
resolve("\nDone");
})
.on("error", (err) => {
reject(err);
})
.on("data", (d) => {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Download ${progress} bytes`);
}
})
.pipe(dest);
});
});
}
Reference:
Drive v3 API Samples
You can see the sample script at download.js.

Send image to Google Drive API from NodeJS

I managed to upload text files to google API with this code
google.drive({
version: 'v3',
auth
})
var media = {
mimeType: 'text/plain',
body: 'text'
}
drive.files.create({
media: media,
fields: 'id'
})
But if i try to upload an image as suggested in documentation i'm getting empty file on the drive.
Trying to do that this way (file exists and has all privileges)
const drive = google.drive({
version: 'v3',
auth
})
var media = {
mimeType: 'image/png',
body: fs.createReadStream(path.resolve(__dirname, '../assets/logo.png'))
}
drive.files.create({
media: media,
fields: 'id'
})
And when I overviewing request in debug console i see that there was no request body.
Please, help.
Below is the whole component which is doing the upload
<template>
<div>
<button class="btn btn-block btn-success mb-3" #click="connect">Syncronize</button>
<h3 class="text-center">
<template v-if="getToken">Sync success</template>
<template v-else>Sync failed</template>
</h3>
<button #click="tryUpload">Test API</button>
</div>
</template>
<script>
import OAuth2 from '../classes/OAuth2'
import { mapActions, mapGetters } from 'vuex'
const {google} = require('googleapis')
export default {
computed: {
...mapGetters([
'getToken'
])
},
methods: {
...mapActions([
'saveToken'
]),
connect () {
(async () => {
let token = await new OAuth2().getToken()
this.saveToken(token)
})()
},
tryUpload () {
const auth = new google.auth.OAuth2(
'....',
'.....',
'http://127.0.0.1:42813/callback'
)
auth.setCredentials(this.getToken)
const drive = google.drive({
version: 'v3',
auth
})
let stream = fs.createReadStream(path.resolve(__dirname, '../assets/logo.png'))
var media = {
mimeType: 'image/png',
body: stream
}
drive.files.create({
media: media,
fields: 'id'
}, function (err, file) {
if (err) {
console.error(err)
} else {
console.log('File Id: ', file.id)
}
})
}
}
}
</script>
Try importing the file with require instead
var media = {
mimeType: 'image/png',
body: require(path.resolve(__dirname, '../assets/logo.png'))
}
As it is, you are uploading the stream itself, not the file being streamed. If you want to use fs, you should try accessing the stream using the callback inside of the stream.on('{event}' function () {}) method.
It's working for me when I upload a file using arrow functions syntax and async/await (at least for me it's more comfortable like this):
// Your tryUpload Function
const tryUpload = async () => {
// ...Previous OAuth workflow
// Build Drive service
const drive = google.drive({version: 'v3', auth});
try {
const data = await uploadFile(drive);
console.log(data);
} catch(err){
console.log(`There was a problem in the promise ---> ${err}`);
}
}
const uploadFile = (drive) =>{
// Set file metadata and data
const fileMetadata = {'name': 'testupload.png'};
const media = {
mimeType: 'image/png',
// If it's in the same dir, just pass 'testupload.png'
body: fs.createReadStream('path/to/testupload.png')
};
// Return the Promise result after completing its task
return new Promise((resolve, reject) => {
// Call Files: create endpoint
return drive.files.create({
resource: fileMetadata,
media: media
},(err, results) => err ? reject(err) : resolve(results))
});
};
Docs
As a guide to help you. I used these docs:
Files: create.
Perform a simple upload.

How to get progress status while uploading files to Google Drive using NodeJs?

Im trying to get progress status values while uploading files to google Drive using nodeJs.
controller.js
exports.post = (req, res) => {
//file content is stored in req as a stream
// 1qP5tGUFibPNaOxPpMbCQNbVzrDdAgBD is the folder ID (in google drive)
googleDrive.makeFile("file.txt","1qP5tGUFibPNaOxPpMbCQNbVzrDdAgBD",req);
};
googleDrive.js
...
makeFile: function (fileName, root,req) {
var fileMetadata = {
'name': fileName,
'mimeType': 'text/plain',
'parents': [root]
};
var media = {
mimeType: 'text/plain',
body: req
};
var r = drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
// r => undefined
console.log("Uploaded: " + r);
}
});
},
...
i followed this link but got always an undefined value
How about this modification?
Modification point:
It used onUploadProgress.
Modified script:
makeFile: function (fileName, root,req) {
var fileMetadata = {
'name': fileName,
'mimeType': 'text/plain',
'parents': [root]
};
var media = {
mimeType: 'text/plain',
body: req
};
var r = drive.files.create({
auth: jwToken,
resource: fileMetadata,
media: media,
fields: 'id'
}, {
onUploadProgress: function(e) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(e.bytesRead.toString());
},
}, function (err, file) {
if (err) {
// Handle error
console.error(err);
} else {
console.log("Uploaded: " + file.data.id);
}
});
},
Note:
If you want to show the progression as "%", please use the file size.
It was confirmed that this script worked at googleapis#33.0.0.
References:
axios
test of google/google-api-nodejs-client
In my environment, I'm using the script like above. But if this didn't work in your environment and if I misunderstand your question, I'm sorry.

Resources