I am adding pdf files created by wkhtmptopdf to archiver.My purpose is to create a zip file of all the pdf's.I am posting the zip to s3 bucket. BUt in the bucket some files are missing in the zip file. Randomly out of all files some files are empty in the zip.
for (var i = 1; i <= dataCount; i++) {
var output_filename = 'testing' +'_'+i+'.pdf';
const html = "sdfsdfsdf";
var obg= wkhtmltopdf(html, wkhtmltopdfOptions);
var results = [];
archive.append(obg, { name: output_filename });
} // closing the for loop
archive.finalize();
Related
I am trying to gzip my folder with the help of Pako library. I couldn't found any related content about it. Can someone explain me how to use pako to gzip directory. I am using it in my lambda function along with EFS.
let bundle = fs.readdirSync(tempDir);
let zippedFile = pako.gzip(bundle);
My folder location looks like this data/temp/
Error
TypeError: strm.input.subarray is not a function
You can use fs.readdirSync() to list files in the directory, then check the stat of each file before compressing it using fs.lstatSync(). I have tested this locally on my mac.
Then you can write the compressed file to your file system.
const fs = require('fs');
const pako = require('pako');
let files = fs.readdirSync('/tmp/');
let fileContent = '';
files.forEach(file => {
let path = '/tmp/' + file;
let stats = fs.lstatSync(path);
if (stats.isFile()) {
let data = fs.readFileSync(path);
fileContent += data;
}
});
let zippedFile = pako.gzip(fileContent);
const stream = fs.createWriteStream('/tmp/dir.gz');
stream.write(zippedFile);
stream.end();
Is there a way to write a script/automate copying over data from an xlsx file to a google sheet file in the same google drive?
I have the xlsx file to auto-sync in my google drive. I want to then connect this data to data studio report for visualization, but that is not compatible with an xlsx file. I need to transfer this data to a google sheet in order to connect, so I was wondering how I could automate the process of moving data from this xlsx file to a gsheet.
There is a project hosted on GitHub that has what you are looking for:
https://gist.github.com/azadisaryev/ab57e95096203edc2741
It creates the new GS file on GDrive from the existing xlsx file on your GDrive.
In your scenario you may want to read data from the newly created GS and put them into a "static" GS (connected to Data Studio). Then you may want to delete (send to trash) newly created GS file.
This code convert XLSX to G Spread Sheet in specific folder. Update if file name (GSS) already exist.
Sure this helps XLSX2GSS.
function XLSX2GSS() {
var sourceFolderId = ""; // Folder ID including source files.
var destinationFolderId = ""; // Folder ID that the converted files are put.
var getFileIds = function (folder, fileList, q) {
var files = folder.searchFiles(q);
while (files.hasNext()) {
var f = files.next();
fileList.push({id: f.getId(), fileName: f.getName().split(".")[0].trim()});
}
var folders = folder.getFolders();
while (folders.hasNext()) getFileIds(folders.next(), fileList, q);
return fileList;
};
var sourceFiles = getFileIds(DriveApp.getFolderById(sourceFolderId), [], "mimeType='" + MimeType.MICROSOFT_EXCEL + "' or mimeType='" + MimeType.MICROSOFT_EXCEL_LEGACY + "'");
var destinationFiles = getFileIds(DriveApp.getFolderById(destinationFolderId), [], "mimeType='" + MimeType.GOOGLE_SHEETS + "'");
var createFiles = sourceFiles.filter(function(e) {return destinationFiles.every(function(f) {return f.fileName !== e.fileName});});
var updateFiles = sourceFiles.reduce(function(ar, e) {
var dst = destinationFiles.filter(function(f) {return f.fileName === e.fileName});
if (dst.length > 0) {
e.to = dst[0].id;
ar.push(e);
}
return ar;
}, []);
if (createFiles.length > 0) createFiles.forEach(function(e) {Drive.Files.insert({mimeType: MimeType.GOOGLE_SHEETS, parents: [{id: destinationFolderId}], title: e.fileName}, DriveApp.getFileById(e.id))});
if (updateFiles.length > 0) updateFiles.forEach(function(e) {Drive.Files.update({}, e.to, DriveApp.getFileById(e.id))});
}
I want to upload files which is present in tempFolder to 2 different buckets of s3.My code works as expected to upload multiple files to foldername1.
tempFolder has 8 files. i have upload all 8 to foldername1 and only last 2 files to my foldername2 in s3.
how can i add them.
function foo() {
let tempFolder = // folder path
return fs.readdirSync(tempFolder ).map((file) => {
let path = path.join(tempFolder , file);
let awsPath1 = // s3 bucket foldername1
let awsPath2 = // s3 bucket foldername2
let total = fs.statSync(path);
return s3Upload(awsPath1, () => fs.createReadStream(path), total.size)
.then(
() => {
s3Uploaded.push("https://aws.amazon.com/s3/bucketname/" + awsPath);
});
});
}
I have an unzipped xlsx file, in it I edit some files to be able to generate a new xlsx file containing new data.
In linux to recompress the file in xlsx I just need to go into the terminal and type
find . -type f | xargs zip ../newfile.xlsx
into the folder where the xlsx files are.
The question now is how can I do this using node.js?
The solution is to compress a direct list of files contained in xlsx, for some reason if we try to compress the folder the file has corrupted.
The code looks like this if you use JSZIP
var fs = require('fs');
var JSZip = require("jszip");
var zip = new JSZip();
var file = [];
file.push("_rels/.rels");
file.push("docProps/core.xml");
file.push("docProps/app.xml");
file.push("docProps/custom.xml");
file.push("[Content_Types].xml");
file.push("xl/_rels/workbook.xml.rels");
file.push("xl/styles.xml");
file.push("xl/pivotTables/_rels/pivotTable3.xml.rels");
file.push("xl/pivotTables/_rels/pivotTable1.xml.rels");
file.push("xl/pivotTables/_rels/pivotTable2.xml.rels");
file.push("xl/pivotTables/pivotTable3.xml");
file.push("xl/pivotTables/pivotTable1.xml");
file.push("xl/pivotTables/pivotTable2.xml");
file.push("xl/workbook.xml");
file.push("xl/worksheets/_rels/sheet2.xml.rels");
file.push("xl/worksheets/_rels/sheet1.xml.rels");
file.push("xl/worksheets/_rels/sheet3.xml.rels");
file.push("xl/worksheets/sheet4.xml");
file.push("xl/worksheets/sheet1.xml");
file.push("xl/worksheets/sheet3.xml");
file.push("xl/worksheets/sheet2.xml");
file.push("xl/sharedStrings.xml");
file.push("xl/pivotCache/_rels/pivotCacheDefinition1.xml.rels");
file.push("xl/pivotCache/pivotCacheDefinition1.xml");
file.push("xl/pivotCache/pivotCacheRecords1.xml");
for (var i = 0; i < file.length; i++) {
zip.file(file[i], fs.readFileSync("/home/user/xlsx_FILES/"+file[i]));
}
zip.generateAsync({type:"blob"}).then(function(content) {
// see FileSaver.js
saveAs(content, "yourfile.xlsx");
});
Take a look at archiver, a compression library for nodejs. The docs for the library look like they are comprehensive. The library also allows you to append archives and take advantage of streaming api's for appending and creating new archives.
Here is an example snippet from their docs which shows how to use the library.
// require modules
var fs = require('fs');
var archiver = require('archiver');
// create a file to stream archive data to.
var output = fs.createWriteStream(__dirname + '/example.zip');
var archive = archiver('zip', {
store: true // Sets the compression method to STORE.
});
// listen for all archive data to be written
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
// good practice to catch this error explicitly
archive.on('error', function(err) {
throw err;
});
// pipe archive data to the file
archive.pipe(output);
I am writing a nodejs script which should do the following:
Download a zip file
Remove the top level directory of the zip file (moving all files one folder up)
Upload the new zip file
Because the zip file is rather large, I would like to rename (or move) the files without unzipping and rezipping the file.
Is that possible?
Yes, it's possible
Using a library like adm-zip
var AdmZip = require('adm-zip');
//create a zip object to hold the new zip files
var newZip = new AdmZip();
// reading archives
var zip = new AdmZip('somePath/download.zip');
var zipEntries = zip.getEntries(); // an array of ZipEntry records
zipEntries.forEach(function(zipEntry) {
var fileName = zipEntry.entryName;
var fileContent = zip.readAsText(fileName)
//Here remove the top level directory
var newFileName = fileName.substring(fileName.indexOf("/") + 1);
newZip.addFile(newFileName, fileContent, '', 0644 << 16);
});
newZip.writeZip('somePath/upload.zip'); //write the new zip
Algorithm
Create a newZip object to temporarily hold files in memory
Read all entries in the downloaded zip. For each entry
Read the fileName. This includes the path
Read the file content using the fileName
Remove the top level directory name to get the newFileName
Add the fileContent in step 2 to the newZip giving it the newFileName from step 3
Finally, write out the newZip to disk giving it a new zipName
Hope that helps
You can use the great jszip library with async-promise style.
import jszip from 'jszip';
import fs from 'fs';
/**
* Move/rename entire directory tree within a zip.
* #param {*} zipFilePath The original zip file
* #param {*} modifiedZipFilePath The path where palace the modified zip
* #param {*} originalDir The original directory to change
* #param {*} destinationDir The new directory to move to.
*/
async function moveDirectory(zipFilePath, modifiedZipFilePath, originalDir, destinationDir) {
// Read zip file bits buffer
const zipFileBuffer = await fs.promises.readFile(zipFilePath);
// Load jszip instance
const zipFile = await jszip.loadAsync(zipFileBuffer);
// Get the original directory entry
const originalDirContent = zipFile.folder(originalDir);
// Walk on all directory tree
originalDirContent.forEach((path, entry) => {
// If it's a directory entry ignore it.
if (entry.dir) {
return;
}
// Extract the file path within the directory tree
const internalDir = path.split(originalDir)[0];
// Build the new file directory in the new tree
const newFileDir = `${destinationDir}/${internalDir}`;
// Put the file in the new tree, with the same properties
zipFile.file(newFileDir, entry.nodeStream(), {
createFolders: true,
unixPermissions: entry.unixPermissions,
comment: entry.comment,
date: entry.date,
});
});
// After all files copied to the new tree, remove the original directory tree.
zipFile.remove(originalDir);
// Generate the new zip buffer
const modifiedZipBuffer = await zipFile.generateAsync({ type: 'nodebuffer' });
// Save the buffer as a new zip file
await fs.promises.writeFile(modifiedZipFilePath, modifiedZipBuffer);
}
moveDirectory('archive.zip', 'modified.zip', 'some-dir/from-dir', 'some-other-dir/to-dir');
This is simply walking on all original directory tree entries and place them in the new directory tree.