I have put together a script that creates a file in a folder on Google Drive, when I make amendments to that file and push it back up it will create a new version (which is obviously correct). What I would like to do is check that the file exists first (by name if I can) and then update that file if exists or create new if it does not.
# index.js
const google = require('googleapis');
const fs = require('fs');
const config = require('./creds.json');
const drive = google.drive('v3');
const targetFolderId = "123456789"
const jwtClient = new google.auth.JWT(
config.client_email,
null,
config.private_key,
['https://www.googleapis.com/auth/drive'],
null
);
jwtClient.authorize((authErr) => {
if (authErr) {
console.log(authErr);
return;
}
const fileMetadata = {
name: './file.xlsx,
parents: [targetFolderId]
};
const media = {
mimeType: 'application/vnd.ms-excel',
body: fs.createReadStream('./file.xlsx' )
};
drive.files.create({
auth: jwtClient,
resource: fileMetadata,
media,
fields: 'id'
}, (err, file) => {
if (err) {
console.log(err);
return;
}
console.log('Uploaded File Id: ', file.data.id);
});
});
Update
As D Levine mentions I can use drive.files.list but I am having problems finding a file. I set up a service account so not sure if that makes a difference. My code is below but all I get at the moment is undefined returned. I would also like to search within specific folders but can't seem to find this option anywhere.
// Check File exists
drive.files.list({
auth: jwtClient,
spaces: 'drive',
q: "name='liverpool_away.xlsx'",
pageSize: 10,
fields: "nextPageToken, files(id, name)"
}, function(err, response) {
if (err) {
console.log('The API returned an error: ' + err);
return;
}
var files = response.files;
console.log("Files: " + files);
if (files.length == 0) {
console.log('No files found.');
} else {
console.log('Files:');
for (var i = 0; i < files.length; i++) {
var file = files[i];
console.log('%s (%s)', file.name, file.id);
}
}
});
You can find if a file exists by searching by name with driveService.files().list(). If you receive 1 result, then that is the file that you were looking for and you can get the file ID. If you receive 0 results, then the file does not exist.
A page describing how to search is here (with example code at the bottom): https://developers.google.com/drive/v3/web/search-parameters
The REST endpoint is documented here: https://developers.google.com/drive/v3/reference/files/list
Related
This question already has answers here:
node.js remove file
(21 answers)
Closed last year.
I have two async functions:
One for creating objects in PostgreSQL and upload files with named in this object.
Another one for deleting this entity and deleting files from folder.
I don't know how to extract filenames from PostgreSQL entity and delete certain files in my 'static' folder
Entity in PostgreSQL looks like:
Car:
{
'name': 'Nissan 350Z',
'description': 'new',
'image': '123wejdsefberfkj.jpg',
'video': '23rusdjf8ioysdfs.mp4'
}
Create function:
Here I get files from form-data, create unique name for files and save it in PostgreSQL, then I save files in "static" folder.
let videoName = uuid.v4() + '.mp4';
let imageName = uuid.v4() + '.jpg';
let {name,description} = req.body;
const {video, image} = req.files;
const car = await Car.create( {name, description, video: videoName,image: imageName})
.then(video.mv(path.resolve(__dirname,'..', 'static', videoName)))
.then(image.mv(path.resolve(__dirname,'..', 'static', imgName)))
Delete function:
Here I need to extract file-names from database and delete them from folder
a bit pseudocode:
async delete(req, res) {
try {
const {id} = req.params;
await Car.findOne({where:{id}})
.then( async data => {
if(data) {
await let videoName = Car.#extract_video_name# ({where: {id}})
.then(mv(path.delete(__dirname,'..','static',videoName)))
await Car.destroy({where:{id}}).then(() => {
return res.json("Car deleted");
})
} else {
return res.json("This Car doesn't exist in database");
}
})
} catch (e) {
console.error(e)
}
}
You can use fs, require it using const fs = require('fs');
You have no npm package to install, it's included in node.js.
This is the code that allows you to save your files into a directory by creating the folder if it doesn't exist and upload the file into it
let path = String(`./directoryName/${fileName}`);
fs.mkdirSync('./directoryName', { recursive: true });
fs.writeFileSync(path, data);
And you can delete the file using
fs.unlink(path, (err) => {
if (err) throw err //handle your error the way you want to;
console.log('path/file.txt was deleted');//or else the file will be deleted
});
);
Refernce : https://nodejs.org/api/fs.html
I am using the Google Drive for Developers Drive API (V3) Nodejs quickstart.
In particular I am concentrating on the following function. Where I have customized the pageSize to 1 for testing. And am calling my function read(file.name);
/**
* Lists the names and IDs of up to 10 files.
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
function listFiles(auth) {
const drive = google.drive({version: 'v3', auth});
drive.files.list({
pageSize: 1, // only find the last modified file in dev folder
fields: 'nextPageToken, files(id, name)',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const files = res.data.files;
if (files.length) {
console.log('Files:');
files.map((file) => {
console.log(`${file.name} (${file.id})`);
read(file.name); // my function here
});
} else {
console.log('No files found.');
}
});
}
// custom code - function to read and output file contents
function read(fileName) {
const readableStream = fs.createReadStream(fileName, 'utf8');
readableStream.on('error', function (error) {
console.log(`error: ${error.message}`);
})
readableStream.on('data', (chunk) => {
console.log(chunk);
})
}
This code reads the file from the Google Drive folder that is synced. I am using this local folder for development. I have found the pageSize: 1 parameter produces the last file that has been modified in this local folder. Therefore my process has been:
Edit .js code file
Make minor edit on testfiles (first txt then gdoc) to ensure it is last modified
Run the code
I am testing a text file against a GDOC file. The filenames are atest.txt & 31832_226114__0001-00028.gdoc respectively. The outputs are as follows:
PS C:\Users\david\Google Drive\Technical-local\gDriveDev> node . gdocToTextDownload.js
Files:
atest.txt (1bm1E4s4ET6HVTrJUj4TmNGaxqJJRcnCC)
atest.txt this is a test file!!
PS C:\Users\david\Google Drive\Technical-local\gDriveDev> node . gdocToTextDownload.js
Files:
31832_226114__0001-00028 (1oi_hE0TTfsKG9lr8Wl7ahGNvMvXJoFj70LssGNFFjOg)
error: ENOENT: no such file or directory, open 'C:\Users\david\Google Drive\Technical-local\gDriveDev\31832_226114__0001-00028'
My question is:
Why does the script read the text file but not the gdoc?
At this point I must 'hard code' the gdoc file extension to the file name, in the function call, to produce the required output as per the text file example eg
read('31832_226114__0001-00028.gdoc');
Which is obviously not what I want to do.
I am aiming to produce a script that will download a large number of gdocs that have been created from .jpg files.
------------------------- code completed below ------------------------
/**
* Lists the names and IDs of pageSize number of files (using query to define folder of files)
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
function listFiles(auth) {
const drive = google.drive({version: 'v3', auth});
drive.files.list({
corpora: 'user',
pageSize: 100,
// files in a parent folder that have not been trashed
// get ID from Drive > Folder by looking at the URL after /folders/
q: `'11Sejh6XG-2WzycpcC-MaEmDQJc78LCFg' in parents and trashed=false`,
fields: 'nextPageToken, files(id, name)',
}, (err, res) => {
if (err) return console.log('The API returned an error: ' + err);
const files = res.data.files;
if (files.length) {
var ids = [ ];
var names = [ ];
files.forEach(function(file, i) {
ids.push(file.id);
names.push(file.name);
});
ids.forEach((fileId, i) => {
fileName = names[i];
downloadFile(drive, fileId, fileName);
});
}
else
{
console.log('No files found.');
}
});
}
/**
* #param {google.auth.OAuth2} auth An authorized OAuth2 client.
*/
function downloadFile(drive, fileId, fileName) {
// make sure you have valid path & permissions. Use UNIX filepath notation.
const filePath = `/test/test1/${fileName}`;
const dest = fs.createWriteStream(filePath);
let progress = 0;
drive.files.export(
{ fileId, mimeType: 'text/plain' },
{ responseType: 'stream' }
).then(res => {
res.data
.on('end', () => {
console.log(' Done downloading');
})
.on('error', err => {
console.error('Error downloading file.');
})
.on('data', d => {
progress += d.length;
if (process.stdout.isTTY) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`Downloading ${fileName} ${progress} bytes`);
}
})
.pipe(dest);
});
}
My question is: Why does the script read the text file but not the gdoc?
This is because you're trying to download a Google Workspace document, only files with binary content can be downloaded using drive.files.get method. For Google Workspace documents you need to use drive.files.exports as documented here
From your code, I'm seeing you're only listing the files, you will need to identify the type of file you want to download, you can use the mimeType field to check if you need to use the exports method vs get, for example, a Google Doc mime type is application/vnd.google-apps.document meanwhile a docx file (binary) would be application/vnd.openxmlformats-officedocument.wordprocessingml.document
Check the following working example:
Download a file from Google Drive
Run in Fusebit
const fs = require("fs");
const getFile = async (drive, fileId, name) => {
const res = await drive.files.get({ fileId, alt: "media" }, { responseType: "stream" });
return new Promise((resolve, reject) => {
const filePath = `/tmp/${name}`;
console.log(`writing to ${filePath}`);
const dest = fs.createWriteStream(filePath);
let progress = 0;
res.data
.on("end", () => {
console.log("🎉 Done downloading file.");
resolve(filePath);
})
.on("error", (err) => {
console.error("🚫 Error downloading file.");
reject(err);
})
.on("data", (d) => {
progress += d.length;
console.log(`🕛 Downloaded ${progress} bytes`);
})
.pipe(dest);
});
};
const fileKind = "drive#file";
let filesCounter = 0;
const drive = googleClient.drive({ version: "v3" });
const files = await drive.files.list();
// Only files with binary content can be downloaded. Use Export with Docs Editors files
// Read more at https://developers.google.com/drive/api/v3/reference/files/get
// In this example, any docx folder will be downloaded in a temp folder.
const onlyFiles = files.data.files.filter(
(file) =>
file.kind === fileKind &&
file.mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
);
const numberOfFilesToDownload = onlyFiles.length;
console.log(`😏 About to download ${numberOfFilesToDownload} files`);
for await (const file of onlyFiles) {
filesCounter++;
console.log(`📁 Downloading file ${file.name}, ${filesCounter} of ${numberOfFilesToDownload}`);
await getFile(drive, file.id, file.name);
}
The answer (as I see it) is that the nodejs script above is running on Windows and therefore must comply with the native OS/file system inherited via the DOS/NT development of Windows. On the other hand, the gdoc extension is a reference created by the Google Drive sync desktop client. And here is the important distinction. The gdoc extension references a file stored on Google Drive (the reference being in the sync folder on a hard drive and the file being in the cloud on Google Drive) Therefore it's not an extension in the usual sense. The usual sense being where the extension is used by a local application as a valid access/read/write file type. So my test function above function read(fileName) won't be able to read the .gdoc in the same way as the .txt extension.
Therefore the correct way to access files on Google Drive from a local application is to use the file's ID. The filename is just a convenient way of labelling the IDs so that the user can meaningfully compare the downloaded copy of the file with the original on Google Drive.
(Refer to the original question) Using the code under the ---------- code completed below --------- I have added these two functions to Google's Nodejs Quickstart Replacing the function listFiles(auth) and adding function downloadFile(drive, fileId, fileName)
The total script file has been used to download multiple files (more than 50 at a time) to my hard drive. This is a useful piece of code in an OCR setup which has a gscript convert .JPG images of historic Electoral Rolls into readable text. These gdocs are messy (still containing the original image and colored fonts of various formats) In downloading as text files the above script cleans them up. Of course images are removed from text files and the fonts are standardized to just upper/lower case text. So, it's more than just a downloader. It's a filter as well.
I hope this of some use to someone.
I am using Drive API v3 (Node.js), to create a Google Doc with some data. Later, I also want to have the possibility of "appending" new data to the existing Google Doc.
I have written the following code to create a new Google Doc in a certain folder:
var content = "Content to be written in file"
var fileMetadata = {
name: filename,
parents: [rootFolderId]
};
var media = {
mimeType: 'application/vnd.google-apps.document',
body: content // In the form of string
};
drive.files.create({
resource: fileMetadata,
multipart: media,
fields: 'id',
})
.then(function (response) {
// Handle the response
console.log(response.data.name, "File created")
},
function (err) {
console.error(err);
})
My question is, how can I create the Doc, and initialise it with a string? I want it to be readable on Drive. Right now, a binary file is getting created, with 'No preview available'.
Also, I want to have a function to update this doc (append) with a string. Like this:
var media = {
mimeType: 'application/vnd.google-apps.document',
body: content_to_be_appended // in the form of string
};
drive.files.update({
fileId: existingDocID,
resource: fileMetadata,
multipart: media,
fields: 'id, name'
})
Any help would be greatly appreciated! Thanks!
I believe your goal as follows.
Your question has the following 2 questions.
You want to know the method for creating new Google Document including the text data.
You want to know the method for adding more text data to the existing Google Document.
You want to achieve this using Drive API with googleapis for Node.js.
You have already been able to get and put the file using Drive API.
Answer for question 1:
In this answer, new Google Document is created by including the text data using Drive API.
Modification points:
In this case, it is required to convert the text to the stream type.
When the text is converted to Google Document, mimeType is required to be included in fileMetadata.
When above points are reflected to your script, it becomes as follows.
Modified script:
From:
var content = "Content to be written in file"
var fileMetadata = {
name: filename,
parents: [rootFolderId]
};
var media = {
mimeType: 'application/vnd.google-apps.document',
body: content // In the form of string
};
To:
const stream = require("stream");
var filename = "sample filename"; // Please set the filename of created Google Document.
var rootFolderId = "root"; // Please set the folder ID.
var content = "Content to be written in file";
var bufferStream = new stream.PassThrough();
bufferStream.end(Uint8Array.from(Buffer.from(content, "binary")));
var fileMetadata = {
name: filename,
parents: [rootFolderId],
mimeType: "application/vnd.google-apps.document",
};
var media = {
mimeType: "text/plain", // <--- Added
body: bufferStream
};
In this case, stream module is used.
Answer for question 2:
In this answer, more text data is added to the existing Google Document using Drive API.
Modification points:
In this case, it is required to do the following flow.
Retrieve all texts data from the existing Google Document.
Add more text data to the retrieved texts.
Update the existing Google Document using the updated text data.
In this case, the method of "Files: update" in Drive API is used.
The sample script is as follows.
Sample script:
const documentId = "###"; // Please set the Google Document ID of the existing Google Document.
drive.files.export(
{
fileId: documentId,
mimeType: "text/plain",
},
{ responseType: "stream" },
(err, { data }) => {
if (err) {
console.log(err);
return;
}
let buf = [];
data.on("data", (e) => buf.push(e));
data.on("end", () => {
const stream = require("stream");
const content = "\n" + "Added text data"; // Here, the text data is added to the existing text in Document.
buf.push(Buffer.from(content, "binary"));
const bufferStream = new stream.PassThrough();
bufferStream.end(Uint8Array.from(Buffer.concat(buf)));
var media = {
body: bufferStream,
};
drive.files.update(
{
fileId: documentId,
resource: {},
media: media,
fields: "id",
},
function (err, file) {
if (err) {
console.error(err);
return;
}
console.log(file.data.id);
}
);
});
}
);
In this sample script, I used const content = "\n" + "Added text data"; for adding more text data. If you don't want to insert the line break for this, please remove "\n".
Note:
In order to add more text data, I think that you can also use Docs API. But in your goal, Drive API is used. So I proposed the method for using Drive API.
References:
Files: create
Class: stream.PassThrough
Files: update
From the Media Uploads example for googleapis#60.0.1, you can create a Google Document with a given title and content inside a given folder with
const drive = google.drive({ version: 'v3', auth });
const filename = '<filename>';
const parentFolderId = '<parent-folder-id>';
const content = '<file-content>';
const requestBody = {
name: filename,
parents: [parentFolderId],
mimeType: 'application/vnd.google-apps.document',
};
const media = {
mimeType: 'text/plain',
body: content,
};
await drive.files.create({
requestBody,
media,
fields: 'id',
});
To perform modifications on the document, best use the Docs API. It offers fine control over the document modifications.
If you're looking for a simple solution to update a Google Document's content using the Drive API, a slightly coarser approach to using the Docs API is
drive = google.drive({ version: 'v3', auth });
const fileId = '<file-id>';
const newContent = '<new content>';
const media = {
mimeType: 'text/plain',
body: newContent,
};
await drive.files.update({
fileId,
media,
});
For appending text to a document using the Drive API, you can use something along the lines of
const drive = google.drive({ version: 'v3', auth });
const fileId = '<file-id>';
const contentToAppend = '<new content>';
const { data: prevContent } = await drive.files.export({
fileId,
mimeType: 'text/plain',
});
const newContent = prevContent + contentToAppend;
const media = {
mimeType: 'text/plain',
body: newContent,
};
await drive.files.update({
fileId,
media,
});
I've been playing with the google sheets API in NodeJS for the past couple of days and I'm having some trouble. I took one of the examples from their docs and edited it to suit my needs, however at one point it started returning an error. The console logs:
The API returned an error: SyntaxError: Unexpected token in JSON at position 0
Here's my code:
function getApplies(auth) {
const sheets = google.sheets({version: 'v4', auth});
sheets.spreadsheets.values.get({
spreadsheetId: '1g5kYIIQy5f-UTSmOatTBPOto13ccfPvsJKQpxT6lhjE',
range: 'A2:I',
}, (err2, res) => {
if (err2) return console.log('The API returned an error: ' + err2);
const rows = res.data.values;
if (rows.length) {
//saving data here...
}
});
}
Any idea what I could be doing wrong here? Thanks!
From this link:
Go to Google Console
Create a new Credential of type service account key
Download your json file
npm install google-spreadsheet#2.0.3
var GoogleSpreadsheet = require('google-spreadsheet');
var creds = require('./client_secret.json');
var doc = new GoogleSpreadsheet('*spreadsheet ID*');
doc.useServiceAccountAuth(creds, function (err) {
doc.getRows(1, function (err, rows) {
if (err) {
console.log(err);
} else {
console.log(rows.length);
console.log(rows);
for (var i=0; i < rows.length; i++) {
console.log(rows[i].TITLE-NAME-OF-YOUR-FIRST-COLUMN, rows[i].TITLE-NAME-OF-YOUR-SECOND-COLUMN);
}
}
});
});
Replace the spreadsheet ID with the ID found in the spreadsheet URL.
Share the sheet with the service account email. It is in your json file.
Hope this helps!!
For v4 of googleapi use:
const { google } = require("googleapis");
const key = require("service_account_credentials.json");
const client = new google.auth.JWT(key.client_email, null, key.private_key, [
"https://www.googleapis.com/auth/spreadsheets",
]);
client.authorize(function (err, tokens) {
const gsapi = google.sheets({ version: "v4", auth: client });
const opt = {
spreadsheetId: "spreadsheetId",
range: "Sheet1!A:T",
};
gsapi.spreadsheets.values.get(opt).then(res => console.log(res));
});
You need to parse your json file like this, first:
const rows = JSON.parse(res.data.values);
I am working on a Meteor application.
Currently, I have a few PDFs on my server. To serve these already existing PDFs directly to the client, I do it this way and it works very well:
Router.route("/file/:fileName", function() {
var fileName = this.params.fileName;
// console.log(process.env.PWD);
var filePath = process.env.PWD + '/' + fileName;
var fs = Meteor.npmRequire('fs');
var data = fs.readFileSync(filePath);
this.response.writeHead(200, {
"Content-Type": "application/pdf",
"Content-Length": data.length
});
this.response.write(data);
this.response.end();
}, {
where: "server"
});
I save these PDFs to Mongo using CollectionFS (Later, I shall generate PDFs and save them. For now, I am just directly saving these already existing PDFs to Mongo as I first want to get the Mongo part to work.).
testCollection = new FS.Collection("testCollection", {
stores: [new FS.Store.GridFS("testCollection")]
});
testCollection.allow({
'insert': function () {
return true;
}
});
var file = new FS.File(process.env.PWD + '/PDFKitExampleServerSide.pdf');
file.encoding = 'binary';
file.name('myPDF.pdf');
var document = testCollection.insert(file);
console.log(document._id);
My question is, after I save these PDFs to Mongo using CollectionFS (like I do above), how do I retrieve and serve these PDFs?
Router.route("/database/:pdfId", function() {
//need help here
}, { where: "server"});
After a lot of searching and trying, I've finally gotten it to work.
Router.route("/database/:pdfId", function() {
var pdfId = this.params.pdfId;
var file = testCollection.findOne({_id: pdfId});
var readable = file.createReadStream("tmp");
var buffer = new Buffer(0);
readable.on("data", function(b) {
buffer = Buffer.concat([buffer, b]);
});
var response = this.response;
readable.on("end", function() {
response.writeHead(200, {
"Content-Type": "application/pdf",
"Content-Length": buffer.length
});
response.write(buffer);
response.end();
});
}, {
where: "server"
});
I know that this question is old, but I found an easier way to store and retrieve PDFs. Apparently if you store your PDFs in the database and they are smaller than 16MB (which is likely in this type of files) the performance is way slower than if you store the files in your server file system.
For doing that, you can use FS.Store.FileSystem instead of FS.Store.GridFS. The following code works for me:
// Client
var pdfStore = new FS.Store.FileSystem("uploadedFiles");
UploadedFiles = new FS.Collection("uploadedFiles", {
stores: [pdfStore],
filter: {
allow: {
extensions: ['pdf','doc','docx','xls','xlsx','ppt','pptx''jpg','png','jpeg']
}
}
});
// Server
var pdfStore = new FS.Store.FileSystem("uploadedFiles", {path: uploadFilesPath});
UploadedFiles = new FS.Collection("uploadedFiles", {
stores: [pdfStore],
filter: {
maxSize: 5242880, // 5MB in bytes
allow: {
extensions: ['pdf','doc','docx','xls','xlsx','ppt','pptx''jpg','png','jpeg']
},
deny: {
extensions: ['exe']
},
onInvalid: function (message) {
if (Meteor.isClient) {
alert(message);
} else {
console.log(message);
}
}
}
});
And then just use this little helper to retrieve the url to the file:
get_uploaded_link: function(id){
console.log(id);
var file = UploadedFiles.findOne({_id: id});
return file.url();}