In my Meteor server app, I am downloading a file from Google Drive using this code,
var dest = fs.createWriteStream('/data/'+data.name);
drive.files.get({
fileId: data.id,
alt: 'media',
auth: jwtClient
})
.on('end', Meteor.bindEnvironment(function() {
}))
.on('error', function(err) {
console.log('Error during download', err);
})
.pipe(dest);
How can I get the progress of the download? For example, i want every 30 seconds to display progress of the download using console.log()
Can I use .on('data')? I am using google drive nodejs v3 provided by Google.
You can get File meta (id, name, size) from drive.files.list with file name, then you can download the file.
Use
Node.js Quickstart for google drive to authenticate.
I am using progress-stream to measure % data received.
var callAfterDownload = function (fileName, callback) {
drive.files.list({
auth: oauth2Client,
pageSize: 1,
q: 'name=\'' + fileName + '\'',
fields: 'nextPageToken, files(id, name, size)'
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err)
callback(['Error while download'])
} else {
var files = response.files
//when only one file is matched we will download
if (files.length === 1) {
var file = files.pop()
console.log('%s (%s)', file.name, file.id)
var dest = fs.createWriteStream(file.name)
var progress = Progress({time:100, length: file.size})
//downloading matched file from drive
drive.files.get({
auth: oauth2Client,
fileId: file.id,
alt: 'media'
}).on('error', function (err) {
console.log('Error during download', err)
callback(['Error while download'])
}).pipe(progress).pipe(dest)
//checking progress of file
progress.on('progress', function(progress) {
console.log('download completed ' +progress.percentage.toFixed(2) + '%')
});
//when write stream has finally written to file
dest.on('finish', callback)
} else {
console.log('EXITING......More than one/no file exist with same name, make sure you have unique file name.')
callback()
}
}
})
}
function downloadDriveFile () {
var fileName = 'testfile.doc'
callAfterDownload(fileName, function (err) {
if(err) throw err
//your logic to do anything with the file
})
}
downloadDriveFile();
Related
This question already has an answer here:
How to download dynamic files from google drive
(1 answer)
Closed 13 days ago.
I'm trying to download all the spreadsheets contained in a folder using a service account.
I cannot find a solution, I hope someone could help me.
I authenticate and I get successfully drive.files.list but then I can't download files.
This is my code
import { google } from "googleapis";
import { privatekey } from "./privatekey";
import { createWriteStream, writeFileSync } from "fs";
let jwtClient = new google.auth.JWT(privatekey.client_email, undefined, privatekey.private_key, [
"https://www.googleapis.com/auth/drive",
]);
//authenticate request
jwtClient.authorize(function (err, tokens) {
if (err) {
console.log(err);
return;
} else {
console.log("Successfully connected");
}
});
const folder_id = FOLDER_ID
let drive = google.drive("v3");
drive.files.list(
{
auth: jwtClient,
q: `'${folder_id}' in parents and trashed=false`,
},
function (err, response) {
if (err) {
console.log("The API returned an error: " + err);
return;
}
var files = response?.data.files;
if (files?.length == 0) return;
files?.forEach(async (file) => {
let fileId = file.id;
fileId == null ? (fileId = undefined) : (fileId = fileId);
//writeFileSync(`./cartella/${file.id}.xlsx`, "");
prova(jwtClient, fileId, file.mimeType);
//await getFileFromStream(jwtClient, fileId, file.mimeType);
});
}
);
function getFileFromStream(auth: any, fileId: any, mimeType: any) {
const destPath = `./cartella/${fileId}.xls`;
const dest = createWriteStream(destPath);
return new Promise(async (resolve, reject) => {
const drive = google.drive({ version: "v3", auth });
drive.files.get({
fileId: fileId,
alt: "media",
}),
(err: any, res: any): void => {
res.data
.on("end", () => {
console.log("Done");
})
.on("error", (error: any) => {
console.log("Error during download", error);
})
.pipe(dest);
};
});
}
function prova(auth: any, fileId: any, mimeType: any) {
const destPath = `./cartella/${fileId}.xls`;
const dest = createWriteStream(destPath);
const drive = google.drive({ version: "v3", auth });
drive.files.export({ fileId: fileId, mimeType: mimeType },{responseType: "stream"}, (err: any, res: any) => {
if (err) {
// handle error
console.log("error: ",err)
} else {
if (res == null) return
res.data
.on("end", function () {
console.log("Done");
})
.on("error", function (err: any) {
console.log("Error during download", err);
})
.pipe(dest);
}})
}
First of all I added the service account to the editors of the folder in google drive
The function getFileFromStream returns a big error, but I think that the most interesting thing is this one
domain: 'global',
reason: 'fileNotDownloadable',
message: 'Only files with binary content can be downloaded. Use Export with Docs Editors files.', locationType: 'parameter',
location: 'alt' } ]
So I tried to use drive.files.export, but the response is
status: 400,
statusText: 'Bad Request',
request: {
responseURL: 'https://www.googleapis.com/drive/v3/files/file_id/export?mimeType=application%2Fvnd.google-apps.spreadsheet'
}
I also tried a different authentication method like the one proposed here:
Setting up Google Drive API on NodeJS using a service account
but it still does't work
What am I doing wrong?
The following method will download files that are not Google Drive mime types. You only need to use export if it is a Google Drive mime types and you need to covert it when its download for example a Google sheet converted to an excel file, or a Google docs file converted to a Microsoft word file.
def download_file(service, file_id):
try:
# Call the Drive v3 API
# Get file name, so we can save it as the same with the same name.
file = service.files().get(fileId=file_id).execute()
file_name = file.get("name")
print(f'File name is: {file_name}')
# Call the Drive v3 API
# get the file media data
request = service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%" % int(status.progress() * 100))
# The file has been downloaded into RAM, now save it in a file
fh.seek(0)
with open(file_name, 'wb') as f:
shutil.copyfileobj(fh, f, length=131072)
except HttpError as error:
# TODO(developer) - Handle errors from drive API.
print(f'An error occurred: {error}')
Solution found here:
How to download dynamic files from google drive
Thank you Tanaike for your kind help
So, I have this:
async function uploadImageToFtp(fileName, path) {
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: process.env.FTP_HOST,
user: process.env.FTP_USER,
password: '123',
secure: false
})
await client.uploadFrom(path, "tables/" + fileName)
} catch (err) {
console.log(err)
}
client.close()
}
fs.readdir('plates', function(err, files) {
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
files.forEach(function(file) {
uploadImageToFtp(file, 'plates/' + file);
console.log(file);
});
});
But I get "too many FTP connections...".
So, how to wait for 1 file to upload and then to continue with seconds etc...?
Thank you!
Use for-loop instead of forEach, and use async/await completely for your example:
fs.readdir('plates', async function (err, files) { // async function, carefully this line, `readdir` still is a callback function
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
for (const file of files) {
await uploadImageToFtp(file, 'plates/' + file); // wait until it done
console.log(file);
}
});
I am trying to get a download progress % for a huge file stored in my Google Drive unit when downloading from my Nodejs script.
So far I have written the code to download, which is working, however the on('data'....) part is never called.
const downloadFile = (file) => {
const fileId = file.id;
const fileName = path.join(basePathForStorage, file.name);
const drive = google.drive({ version: 'v3', authorization });
let progress = 0;
return new Promise((resolve, reject) => {
drive.files.get(
{
auth: authorization,
fileId: fileId,
alt: 'media'
},
{ responseType: "arraybuffer" },
function (err, { data }) {
fs.writeFile(fileName, Buffer.from(data), err => {
// THIS PART DOES NOTHING
data.on('data',(d)=>{
progress += d.length;
console.log(`Progress: ${progress}`)
})
// --------------------
if (err) {
console.log(err);
return reject(err);
}
return resolve(fileName)
});
}
);
});
}
Looks like I can't find the way to show the progess of the download by calling on('data'....)...wondering now if this is the correct way to do this, or if this is even possible.
I tried putting the on('data'....) code as it is now inside the writeFile function but also inside the callback from drive.files.get and nothing works.
Here it comes some code sample to do that,
this example has three parts that need to be mentioned:
Create a stream to track our download progress
Create a method to get the file size
Create an event emitter to send back our progress to our FE
So we will get the following:
const downloadFile = async(file) => {
const fileId = file.id
const fileName = path.join(basePathForStorage, file.name)
let progress = 0
/**
* ATTENTION: here you shall specify where your file will be saved, usually a .temp folder
* Here we create the stream to track our download progress
*/
const fileStream = fs.createWriteStream(path.join(__dirname, './temp/', filename))
const fileSize = await getFileSize(file)
// In here we listen to the stream writing progress
fileStream.on('data', (chunk) => {
progress += chunk.length / fileSize
console.log('progress', progress)
})
const drive = google.drive({
version: 'v3',
authorization
})
drive.files.get({
auth: authorization,
fileId: fileId,
alt: 'media'
}, {
responseType: "stream"
},
(err, { data }) =>
data
.on('end', () => console.log('onCompleted'))
.on('error', (err) => console.log('onError', err))
.pipe(fileStream)
)
}
The method to retrieve the file size:
const getFileSize = ({ fileId: id }) => {
const drive = google.drive({
version: 'v3',
authorization
})
return new Promise((resolve, reject) =>
drive.files.get({
auth: authorization,
fileId
}, (err, metadata) {
if (err) return reject(err)
else resolve(metadata.size)
})
}
This code sample give you the ability to get partial updates from your file download as you're creating a write stream (nodejs#createWriteStream)
So you will be able to track your file downloading progress.
But, still you have to continuosly send these changes to your client ( FE ).
So, you could create your own EventEmitter to track that.
And now our sample will be enchanced with the following:
In our endpoint:
import { EventEmitter } from 'events'
router.post('/myEndpoint', (req, res) => {
res.writeHead(200, { 'Content-Type': 'text/plain' })
const progressEvent = new EventEmitter()
progressEvent.on('progress', (progress) => {
if (progress === 100)
res.end()
// So, your FE side will be receiving this message continuosly
else res.write(`{ progress: ${ progress } }`)
})
const file = req.body // or where you're getting your file from
downloadFile(file, progressEvent)
})
In our download method:
const downloadFile = async(file, progressEvent) => {
.
.
.
fileStream.on('data', (chunk) => {
progress += chunk.length / fileSize
progressEvent.emit('progress', progress)
.
.
.
I'm trying to download images from a Google share Drive using the API v3. The download itself will succeed but the image can't be seen. Opening the image from the MacOS finder just results in a spinner.
I started using the example from the documentation (here: https://developers.google.com/drive/api/v3/manage-downloads):
const drive = google.drive({version: 'v3', auth});
// ....
var fileId = '0BwwA4oUTeiV1UVNwOHItT0xfa2M';
var dest = fs.createWriteStream('/tmp/photo.jpg');
drive.files.get({
fileId: fileId,
alt: 'media'
})
.on('end', function () {
console.log('Done');
})
.on('error', function (err) {
console.log('Error during download', err);
})
.pipe(dest);
however that fails because the .on() method doesn't exist. The exact error is "TypeError: drive.files.get(...).on is not a function"
The .get() method returns a promise. The response of the promise contains data that, depending on the config is either a stream, a blob or arraybuffer. For all options, when I write the response data to a file, the file itself becomes unviewable and has the wrong size. The actual code (typescript, node.js) for the arraybuffer example is below. Similar code for blob (with added name and modifiedDate) and for stream give the same result.
const downloader = googleDrive.files.get({
fileId: file.id,
alt: 'media',
}, {
responseType: 'arraybuffer',
});
return downloader
.then((response) => {
const targetFile = file.id + '.' + file.extension;
fs.writeFileSync(targetFile, response.data);
return response.status;
})
.catch((response) => {
logger.error('Error in Google Drive service download: ' + response.message);
return response.message;
}
);
}
So the questions are:
what is the correct way to handle a download through Google Drive API v3 ?
do I need to handle any formatting of the response data ?
All help greatly appreciated!
Thanks
You want to download a file from Google Drive using googleapis with Node.js.
You have already been able to use Drive API.
If my understanding is correct, how about this answer?
Pattern 1:
In this pattern, arraybuffer is used for responseType.
Sample script:
const drive = google.drive({ version: "v3", auth });
var fileId = '###'; // Please set the file ID.
drive.files.get(
{
fileId: fileId,
alt: "media"
},
{ responseType: "arraybuffer" },
function(err, { data }) {
fs.writeFile("sample.jpg", Buffer.from(data), err => {
if (err) console.log(err);
});
}
);
In this case, Buffer.from() is used.
Pattern 2:
In this pattern, stream is used for responseType.
Sample script:
const drive = google.drive({ version: "v3", auth });
var fileId = '###'; // Please set the file ID.
var dest = fs.createWriteStream("sample.jpg");
drive.files.get(
{
fileId: fileId,
alt: "media"
},
{ responseType: "stream" },
function(err, { data }) {
data
.on("end", () => {
console.log("Done");
})
.on("error", err => {
console.log("Error during download", err);
})
.pipe(dest);
}
);
Note:
If an error occurs, please use the latest version of googleapis.
From your question, it seems that you have already been able to retrieve the file you want to download using your request, while the file content cannot be opened. But if an error occurs, please try to add supportsAllDrives: true and/or supportsTeamDrives: true in the request.
References:
Download files
google-api-nodejs-client/samples/drive/download.js
If I misunderstood your question and this was not the direction you want, I apologize.
Posting a third pattern for completeness using async/await and including teamdrive files.
async function downloadFile(drive: Drive, file: Schema$File, localDir: string = "/tmp/downloads") {
if (!fs.existsSync(localDir)) {
fs.mkdirSync(localDir)
}
const outputStream = fs.createWriteStream(`${localDir}/${file.name}`);
const { data } = await drive.files.get({
corpora: 'drive',
includeItemsFromAllDrives: true,
supportsAllDrives: true,
fileId: file.id,
alt: "media",
}, {
responseType: 'stream',
})
await pipeline(data, outputStream)
console.log(`Downloaded file: ${localDir}/${file.name}`)
}
If someone is looking for a solution is 2023, here you go!
const downloadFile = async (file) => {
const dirPath = path.join(process.cwd(), '/images');
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
const filePath = `${dirPath}/${file.name}.jpg`;
const destinationStream = fs.createWriteStream(filePath);
try {
const service = await getService();
const { data } = await service.files.get(
{ fileId: file.id, alt: 'media' },
{ responseType: 'stream' }
);
return new Promise((resolve, reject) => {
data
.on('end', () => {
console.log('Done downloading file.');
resolve(filePath);
})
.on('error', (err) => {
console.error('Error downloading file.');
reject(err);
})
.pipe(destinationStream);
});
} catch (error) {
throw error;
}
};
I'm trying to delete a pdf immediately after is has been generated in node.js. The generated pdf is sent as an email attachment, uploaded to dropbox and then deleted from the local file system. But as i try to delete it , it does not delete it and it does not send the email either. The pdf is created using html-pdf. Here is my code :
if (result) {
var filename = user.number+ ".pdf";
var path = './public/files/'+filename ;
var options = { filename: path, format: 'Legal', orientation: 'portrait', directory: './public/files/',type: "pdf" };
html = result;
pdf.create(html, options).toFile(function(err, res) {
if (err) return console.log(err);
console.log(res);
});
var dbx = new dropbox({ accessToken: mytoken });
fs.readFile( path,function (err, contents) {
if (err) {
console.log('Error: ', err);
}
dbx.filesUpload({ path: "/"+filename ,contents: contents })
.then(function (response) {
console.log("done")
console.log(response);
})
.catch(function (err) {
console.log(err);
});
});
var mailOptions = {
from: 'xyz', // sender address
to: user.email, // list of receivers
subject: 'Confirmation received', // Subject line
attachments : [{
filename: filename,
path : path
}]
};
transporter.sendMail(mailOptions, (error, info) => {
if (error) {
return console.log(error);
}
console.log('Message %s sent: %s', info.messageId, info.response);
});
fs.unlinkSync(path); // even tried fs.unlink , does not delete file
// fs.unlinkSync(someother file); this one works
}
So when i do fs.unlink' orfs.unlinkSync`, if the file is already there it works but the files that is generated as in path doesn't get deleted.
NodeJs is asynchronous so you need to handle every blocks properly. your code shows that some times before completely creating the PDF itself the file upload to dropbox will start if the PDF creation is slow.
And deletion of the PDF file happens before the mail sending, so you get some err but you did not logged you error in fs.unlink(). Divide you code as blocks and use callback for better performance and flow.
Your code should be like this to work properly..
if (result) {
var filename = user.number+ ".pdf";
var path = './public/files/'+filename ;
var options = { filename: path, format: 'Legal', orientation: 'portrait', directory: './public/files/',type: "pdf" };
html = result;
//Generate the PDF first
pdf.create(html, options).toFile(function(err, res) {
if (err){
return console.log(err);
} else {
//If success then read the PDF file and then upload to dropbox
var dbx = new dropbox({ accessToken: mytoken });
fs.readFile( path,function (err, contents) {
if (err) {
console.log('Error: ', err);
} else {
dbx.filesUpload({path: "/"+filename ,contents: contents }).then(function (response) {
// Once the file upload is done then send mail
console.log("done")
sendMail('xyz', user.email, 'Confirmation received', filename, path, function(err, result){
// once mail is successful then delete the file finally
fs.unlinkSync(path); //if you need you can use callback with this for confirmation of deletion
});
}).catch(function(err) {
console.log(err);
});
}
});
}
});
function sendMail(sender, receiver, subject, filename, path, callback){
var mailOptions = {
from: sender, // sender address
to: receiver, // list of receivers
subject: subject, // Subject line
attachments : [{
filename: filename,
path : path
}]
};
transporter.sendMail(mailOptions, (err, info) => {
if (error) {
callback(err, null);
} else {
console.log('Message %s sent: %s', info.messageId, info.response);
callback(null, info)
}
});
}
}