read files and read directory in node.js - node.js

I want to know whtether the read file and read directory functions - fs.readdir(path, callback) and fs.readFile(path, options, callback) have similar functions without callback.Here, I first read all the files in given directory, and loop through all the files and upload their content to S3 bucket.
Please see the working code below.
const s3Upload = async (req, res) => {
const directoryName = "MAXIS_GAMING/Daily/"
var data = {}
let files = {}
await readFiles1(directoryName)
}
const readFiles1 = async(dirname) => {
let _files
fs.readdir(dirname, (err, files) => {
// On error, show it and return
if(err) return console.error(err);
// files is an array containing the names of all entries
// in the directory, excluding '.' (the directory itself)
// and '..' (the parent directory).
// Display directory entries
console.log(files.join(' '));
files.forEach(function(filename){
fs.readFile(dirname + filename, 'utf-8', function(err, content){
if(err) {
// onError(err);
throw err
return;
}
console.log('cont..............................',content)
console.log('filename', filename)
//await
uploadFiles(filename, content)
//onFileContent(filename, content);
})
})
})
}
const uploadFiles = async (fileName, fileContent) => {
console.log('in uploadd..........')
const GLOBAL_ACCESS_KEY_ID = 'AKIDAQWZX6B3XUBDIFHLPC5LYFTJF15XPIQ';
const GLOBAL_SECRET_ACCESS_KEY = 'Sv4Fe4h4QgErG5XoZbgeC63oczkdW3bMQfC0jvyR8bPbJ9Y97k+'
const GLOBAL_DEFAULT_REGION = 'ap-southeast-1';
const S3_IMAGE_BUCKET ='max-stg-image/stage/reports'//"max-stg-image";
const S3_IMAGE_PATH = "stage";
AWS.config.update({
accessKeyId: GLOBAL_ACCESS_KEY_ID,
secretAccessKey: GLOBAL_SECRET_ACCESS_KEY,
region: GLOBAL_DEFAULT_REGION,
});
const s3 = new AWS.S3()
const bucket = new AWS.S3()
const params = {
Bucket: S3_IMAGE_BUCKET,
Key: fileName,
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
}
app.get('/home/s3Upload', s3Upload)

You can do something like this:
import { readdir } from 'fs/promises';
//or with require
const readdir = require('fs/promises').readdir;
try {
const files = await readdir(path);
for (const file of files)
console.log(file);
} catch (err) {
console.error(err);
}
Check here all the promise API provided by FS in Node.js.

Related

How to download all files in aws S3folder in local machine using node js

I have an S3 bucket which has a folder with some files I want to download all the files in that folder to the local machine folder I tried for the single file it's working how to download multiple files.
As per below code in key folderA has 10 files I want to download all the ten to localfolder directory which I mentioned in s3.getObject(params).createReadStream().pipe(ws);
My code :
const downloadObject = () => {
var params = { Bucket: "Sample", Key:"folderA/"};
const ws = fs.createWriteStream(`${__dirname}/localfolder/`);
const s3Stream = s3.getObject(params).createReadStream().pipe(ws);
s3Stream.on("error", (err) => {
ws.end();
});
s3Stream.on("close", () => {
console.log(`downloaded successfully from s3 at ${new Date()}`);
ws.end();
});
};
expected output:
s3 -> bucket/folderA/<10 files>
localmachine -> localfolder/<need all 10 files in local>
There is quite alot to it,
Firstly you would need to list all buckets, then loop over all the buckets (if you only want one fine). Create a local directory if not found etc.
Then find out all files in the bucket and then loop over them, on each path you the get the object and store it.
Here is how would do it with the minio js client (the calls would be the same) tweak it to your needs obviously the folder paths would be different.
/**
* S3 images pull script
*/
const fs = require('fs')
const path = require('path')
const util = require('util')
const readFile = util.promisify(fs.readFile)
const writeFile = util.promisify(fs.writeFile)
//
const rootPath = path.join(__dirname, '..')
const publicPath = path.join(rootPath, 'public', 'images')
//
require('dotenv').config({
path: path.join(rootPath, '.env')
})
// minio client S3
const s3 = new(require('minio')).Client({
endPoint: process.env.S3_HOST,
port: parseInt(process.env.S3_PORT, 10),
useSSL: process.env.S3_USE_SSL === 'true',
accessKey: process.env.S3_ACCESS_KEY,
secretKey: process.env.S3_ACCESS_SECRET,
region: process.env.S3_REGION
})
/**
* Functions
*/
const mkdir = dirPath => {
dirPath.split(path.sep).reduce((prevPath, folder) => {
const currentPath = path.join(prevPath, folder, path.sep);
if (!fs.existsSync(currentPath)) {
fs.mkdirSync(currentPath);
}
return currentPath
}, '')
}
// list objects in bucket
const listObjects = bucket => new Promise(async (resolve, reject) => {
//
bucket.objects = []
bucket.total_objects = 0
bucket.total_size = 0
//
let stream = await s3.listObjectsV2(bucket.name, '', true)
//
stream.on('data', obj => {
if (obj && (obj.name || obj.prefix)) {
bucket.objects.push(obj)
bucket.total_objects++
bucket.total_size = bucket.total_size + obj.size
}
})
//
stream.on('end', () => resolve(bucket))
stream.on('error', e => reject(e))
})
// get an objects data
const getObject = (bucket, name) => new Promise((resolve, reject) => {
s3.getObject(bucket, name, (err, stream) => {
if (err) reject(err)
//
let chunks = []
stream.on('data', chunk => chunks.push(chunk))
stream.on('end', () => resolve(Buffer.concat(chunks || [])))
stream.on('error', e => reject(e))
})
})
/**
*
*/
async function main() {
// get buckets
console.log(`Fetching buckets from: ${process.env.S3_HOST}`)
let buckets = []
try {
buckets = await s3.listBuckets()
console.log(buckets.length + ' buckets found')
} catch (e) {
return console.error(e)
}
// create local folders if not exists
console.log(`Creating local folders in ./api/public/images/ if not exists`)
try {
for (let bucket of buckets) {
//
bucket.local = path.join(publicPath, bucket.name)
try {
await fs.promises.access(bucket.local)
} catch (e) {
if (e.code === 'ENOENT') {
console.log(`Creating local folder: ${bucket.local}`)
await fs.promises.mkdir(bucket.local)
} else
bucket.error = e.message
}
}
} catch (e) {
return console.error(e)
}
// fetch all bucket objects
console.log(`Populating bucket objects`)
try {
for (let bucket of buckets) {
bucket = await listObjects(bucket)
}
} catch (e) {
console.log(e)
}
// loop over buckets and download all objects
try {
for (let bucket of buckets) {
console.log(`Downloading bucket: ${bucket.name}`)
// loop over and download
for (let object of bucket.objects) {
// if object name has prefix
let dir = path.dirname(object.name)
if (dir !== '.') {
try {
await fs.promises.access(path.join(bucket.local, dir))
} catch (e) {
if (e.code === 'ENOENT') {
console.log(`Creating local folder: ${bucket.local}`)
mkdir(path.join(bucket.local, dir))
}
}
}
//
console.log(`Downloading object[${bucket.name}]: ${object.name}`)
await writeFile(path.join(bucket.local, object.name), await getObject(bucket.name, object.name))
}
}
console.log(`Completed!`)
} catch (e) {
console.log(e)
}
}
main()

AWS S3 Loading many files

I need to upload a lot of files (about 65.000) splitted in subdirectory.
I tried to iterate and load every single file like this:
const fs = require("fs");
const path = require("path");
const async = require("async");
const AWS = require("aws-sdk");
const readdir = require("recursive-readdir");
const slash = require("slash");
const { BUCKET, KEY, SECRET } = process.env;
const rootFolder = path.resolve(__dirname, "./");
const uploadFolder = "./test_files/15";
const s3 = new AWS.S3({
signatureVersion: "v4",
accessKeyId: KEY,
secretAccessKey: SECRET,
});
function getFiles(dirPath) {
return fs.existsSync(dirPath) ? readdir(dirPath) : [];
}
async function deploy(upload) {
if (!BUCKET || !KEY || !SECRET) {
throw new Error("you must provide env. variables: [BUCKET, KEY, SECRET]");
}
const filesToUpload = await getFiles(path.resolve(__dirname, upload));
return new Promise((resolve, reject) => {
async.eachOfLimit(
filesToUpload,
10,
async.asyncify(async (file) => {
const Key = file.replace(rootFolder + path.sep, "");
console.log(`uploading: [${slash(Key)}]`);
var options = { partSize: 5 * 1024 * 1024, queueSize: 4 };
return new Promise((res, rej) => {
s3.upload(
{
Key: slash(Key),
Bucket: BUCKET,
Body: fs.readFileSync(file),
},
(err) => {
if (err) {
return rej(new Error(err));
}
res({ result: true });
}
);
});
}),
(err) => {
if (err) {
return reject(new Error(err));
}
resolve({ result: true });
}
);
});
}
deploy(uploadFolder)
.then(() => {
console.log("task complete");
process.exit(0);
})
.catch((err) => {
console.error(err);
process.exit(1);
});
but after a considerable number of uploads i have this:
Error: Error: NetworkingError: connect ETIMEDOUT IP_S3_AWS
I need to upload this set of files from ec2 instance (because its a result of a image processing). I have this behavior from my pc, i don't know if from ec2 have the same problem.
I have considered the way of zip all and upload but i need to keep the original directory structure.
I accept also new way to resolve the problem.
Sorry for my bad english.
It would probably be much simpler to use the AWS CLI aws s3 sync command instead of building this yourself.

Recursively upload files to S3, how to detect if finished?

with a little help I've built an S3 uploader using Node.JS
It all works great and the files get there, they're set correctly and have the right permissions, but i'm stumped on how to detect whether the process has finished.
const async = require('async');
const AWS = require('aws-sdk');
const mime = require('mime');
const fs = require('fs');
const path = require("path");
require('dotenv').config();
const uploadDirToS3 = function(uploadPath) {
// instantiate aws object for s3
var s3 = new AWS.S3();
// async version
function walk(currentDirPath, callback) {
fs.readdir(currentDirPath, function (err, files) {
if (err) {
throw new Error(err);
}
files.forEach(function (name) {
var filePath = path.join(currentDirPath, name);
var stat = fs.statSync(filePath);
if (stat.isFile()) {
callback(filePath, stat);
} else if (stat.isDirectory()) {
walk(filePath, callback);
}
});
});
}
walk(uploadPath, function(filePath) {
fs.readFile(filePath, function (err, data) {
if (err) { throw err; }
// get content-type (html,jpeg,gif,etc...)
var metaData = mime.getType(filePath)
// set bucket, key (filename), body (file),
// public read-only and content-type
var params = {
Bucket: process.env.AWS_BUCKET,
Key: filePath,
Body: data,
ACL: 'public-read',
ContentType: metaData
};
// upload file to s3
s3.putObject(params, function(err, data) {
if (err) {
console.log(err)
} else {
console.log("Successfully uploaded "+filePath);
}
});
});
})
}
uploadDirToS3("./media/media-1517245218111")
Could it literally be a case of checking wether a callback exists and 'break;' ...ing out of the loop?
Any ideas?
You need to use IterateOver Pattern.
When you find a file to copy, increment a variable and when S3 copy is done, track with another variable that it is copied.
When the totalfind == totalcopied, then initiate the callback from the calling function.
function WaterfallOver(list, iterator, callback) {
var nextItemIndex = 0; //keep track of the index of the next item to be processed
function report() {
nextItemIndex++;
// if nextItemIndex equals the number of items in list, then we're done
if(nextItemIndex === list.length)
callback();
else
// otherwise, call the iterator on the next item
iterator(list[nextItemIndex], report);
}
// instead of starting all the iterations, we only start the 1st one
iterator(list[0], report);
}
Hope it helps.

Upload a file to Google Cloud, in a specific directory

How to upload a file on Google Cloud, in a specific bucket directory (e.g. foo)?
"use strict";
const gcloud = require("gcloud");
const PROJECT_ID = "<project-id>";
let storage = gcloud.storage({
projectId: PROJECT_ID,
keyFilename: 'auth.json'
});
let bucket = storage.bucket(`${PROJECT_ID}.appspot.com`)
bucket.upload("1.jpg", (err, file) => {
if (err) { return console.error(err); }
let publicUrl = `https://firebasestorage.googleapis.com/v0/b/${PROJECT_ID}.appspot.com/o/${file.metadata.name}?alt=media`;
console.log(publicUrl);
});
I tried:
bucket.file("foo/1.jpg").upload("1.jpg", ...)
But there's no upload method there.
How can I send 1.jpg in the foo directory?
In Firebase, on the client side, I do:
ref.child("foo").put(myFile);
bucket.upload("1.jpg", { destination: "YOUR_FOLDER_NAME_HERE/1.jpg" }, (err, file) => {
//Do something...
});
This will put 1.jpg in the YOUR_FOLDER_NAME_HERE-folder.
Here is the documentation. By the way, gcloud is deprecated and you should use google-cloud instead.
UPDATE 2020
according to google documentation:
const { Storage } = require('#google-cloud/storage');
const storage = new Storage()
const bucket = storage.bucket('YOUR_GCLOUD_STORAGE_BUCKET')
const blob = bucket.file('youFolder/' + 'youFileName.jpg')
const blobStream = blob.createWriteStream({
resumable: false,
gzip: true,
public: true
})
blobStream.on('error', (err) => {
console.log('Error blobStream: ',err)
});
blobStream.on('finish', () => {
// The public URL can be used to directly access the file via HTTP.
const publicUrl = ('https://storage.googleapis.com/'+ bucket.name + '/' + blob.name)
res.status(200).send(publicUrl);
});
blobStream.end(req.file.buffer)//req.file is your original file
Here you go...
const options = {
destination: 'folder/new-image.png',
resumable: true,
validation: 'crc32c',
metadata: {
metadata: {
event: 'Fall trip to the zoo'
}
}
};
bucket.upload('local-image.png', options, function(err, file) {
// Your bucket now contains:
// - "new-image.png" (with the contents of `local-image.png')
// `file` is an instance of a File object that refers to your new file.
});
If accessing from the same project projectId , keyFilename,.. not required,I use the below code for both upload and download , it works fine.
// Imports the Google Cloud client library
const Storage = require('#google-cloud/storage');
const storage = new Storage();
var destFilename = "./test";
var bucketName = 'cloudtesla';
var srcFilename = 'test';
const options = {
destination: destFilename,
};
//upload file
console.log("upload Started");
storage.bucket(bucketName).upload(srcFilename, {}, (err, file) => {
if(!err)
console.log("upload Completed");
else
console.log(err);
});
//Download file
console.log("Download Started");
storage
.bucket(bucketName)
.file(srcFilename)
.download(options)
.then(() => {
console.log("Download Completed");
})
.catch(err => {
console.error('ERROR:', err);
});
To upload inside specific directory in .NET Core, use
var uploadResponse= await storageClient.UploadObjectAsync(bucketName, $"{foldername}/"+fileName, null, memoryStream);
This should upload your file 'fileName' inside folder 'foldername' in the bucket
I think just adding foo/ to the filename should work, like bucket.upload("foo/1.jpg", (err, file) ... In GCS, directories just a matter of having a '/' in the file name.
If you want to use async-await while uploading files into storage buckets the callbacks won't do the job, Here's how I did it.
async function uploadFile() {
const destPath = 'PATH_TO_STORAGE/filename.extension';
await storage.bucket("PATH_TO_YOUR_BUCKET").upload(newFilePath, {
gzip: true,
destination: destPath,
});
}
Hope it helps someone!

Create a zip file on S3 from files on S3 using Lambda Node

I need to create a Zip file that consists of a selection of files (videos and images) located in my s3 bucket.
The problem at the moment using my code below is that I quickly hit the memory limit on Lambda.
async.eachLimit(files, 10, function(file, next) {
var params = {
Bucket: bucket, // bucket name
Key: file.key
};
s3.getObject(params, function(err, data) {
if (err) {
console.log('file', file.key);
console.log('get image files err',err, err.stack); // an error occurred
} else {
console.log('file', file.key);
zip.file(file.key, data.Body);
next();
}
});
},
function(err) {
if (err) {
console.log('err', err);
} else {
console.log('zip', zip);
content = zip.generateNodeStream({
type: 'nodebuffer',
streamFiles:true
});
var params = {
Bucket: bucket, // name of dest bucket
Key: 'zipped/images.zip',
Body: content
};
s3.upload(params, function(err, data) {
if (err) {
console.log('upload zip to s3 err',err, err.stack); // an error occurred
} else {
console.log(data); // successful response
}
});
}
});
Is this possible using Lambda, or should I look at a different
approach?
Is it possible to write to a compressed zip file on the fly, therefore eliminating the memory issue somewhat, or do I need to have the files collected before compression?
Any help would be much appreciated.
Okay, I got to do this today and it works. Direct Buffer to Stream, no disk involved. So memory or disk limitation won't be an issue here:
'use strict';
const AWS = require("aws-sdk");
AWS.config.update( { region: "eu-west-1" } );
const s3 = new AWS.S3( { apiVersion: '2006-03-01'} );
const _archiver = require('archiver');
//This returns us a stream.. consider it as a real pipe sending fluid to S3 bucket.. Don't forget it
const streamTo = (_bucket, _key) => {
var stream = require('stream');
var _pass = new stream.PassThrough();
s3.upload( { Bucket: _bucket, Key: _key, Body: _pass }, (_err, _data) => { /*...Handle Errors Here*/ } );
return _pass;
};
exports.handler = async (_req, _ctx, _cb) => {
var _keys = ['list of your file keys in s3'];
var _list = await Promise.all(_keys.map(_key => new Promise((_resolve, _reject) => {
s3.getObject({Bucket:'bucket-name', Key:_key})
.then(_data => _resolve( { data: _data.Body, name: `${_key.split('/').pop()}` } ));
}
))).catch(_err => { throw new Error(_err) } );
await new Promise((_resolve, _reject) => {
var _myStream = streamTo('bucket-name', 'fileName.zip'); //Now we instantiate that pipe...
var _archive = _archiver('zip');
_archive.on('error', err => { throw new Error(err); } );
//Your promise gets resolved when the fluid stops running... so that's when you get to close and resolve
_myStream.on('close', _resolve);
_myStream.on('end', _resolve);
_myStream.on('error', _reject);
_archive.pipe(_myStream); //Pass that pipe to _archive so it can push the fluid straigh down to S3 bucket
_list.forEach(_itm => _archive.append(_itm.data, { name: _itm.name } ) ); //And then we start adding files to it
_archive.finalize(); //Tell is, that's all we want to add. Then when it finishes, the promise will resolve in one of those events up there
}).catch(_err => { throw new Error(_err) } );
_cb(null, { } ); //Handle response back to server
};
I formated the code according to #iocoker.
main entry
// index.js
'use strict';
const S3Zip = require('./s3-zip')
const params = {
files: [
{
fileName: '1.jpg',
key: 'key1.JPG'
},
{
fileName: '2.jpg',
key: 'key2.JPG'
}
],
zippedFileKey: 'zipped-file-key.zip'
}
exports.handler = async event => {
const s3Zip = new S3Zip(params);
await s3Zip.process();
return {
statusCode: 200,
body: JSON.stringify(
{
message: 'Zip file successfully!'
}
)
};
}
Zip file util
// s3-zip.js
'use strict';
const fs = require('fs');
const AWS = require("aws-sdk");
const Archiver = require('archiver');
const Stream = require('stream');
const https = require('https');
const sslAgent = new https.Agent({
KeepAlive: true,
rejectUnauthorized: true
});
sslAgent.setMaxListeners(0);
AWS.config.update({
httpOptions: {
agent: sslAgent,
},
region: 'us-east-1'
});
module.exports = class S3Zip {
constructor(params, bucketName = 'default-bucket') {
this.params = params;
this.BucketName = bucketName;
}
async process() {
const { params, BucketName } = this;
const s3 = new AWS.S3({ apiVersion: '2006-03-01', params: { Bucket: BucketName } });
// create readstreams for all the output files and store them
const createReadStream = fs.createReadStream;
const s3FileDwnldStreams = params.files.map(item => {
const stream = s3.getObject({ Key: item.key }).createReadStream();
return {
stream,
fileName: item.fileName
}
});
const streamPassThrough = new Stream.PassThrough();
// Create a zip archive using streamPassThrough style for the linking request in s3bucket
const uploadParams = {
ACL: 'private',
Body: streamPassThrough,
ContentType: 'application/zip',
Key: params.zippedFileKey
};
const s3Upload = s3.upload(uploadParams, (err, data) => {
if (err) {
console.error('upload err', err)
} else {
console.log('upload data', data);
}
});
s3Upload.on('httpUploadProgress', progress => {
// console.log(progress); // { loaded: 4915, total: 192915, part: 1, key: 'foo.jpg' }
});
// create the archiver
const archive = Archiver('zip', {
zlib: { level: 0 }
});
archive.on('error', (error) => {
throw new Error(`${error.name} ${error.code} ${error.message} ${error.path} ${error.stack}`);
});
// connect the archiver to upload streamPassThrough and pipe all the download streams to it
await new Promise((resolve, reject) => {
console.log("Starting upload of the output Files Zip Archive");
streamPassThrough.on('close', resolve());
streamPassThrough.on('end', resolve());
streamPassThrough.on('error', reject());
archive.pipe(streamPassThrough);
s3FileDwnldStreams.forEach((s3FileDwnldStream) => {
archive.append(s3FileDwnldStream.stream, { name: s3FileDwnldStream.fileName })
});
archive.finalize();
}).catch((error) => {
throw new Error(`${error.code} ${error.message} ${error.data}`);
});
// Finally wait for the uploader to finish
await s3Upload.promise();
}
}
The other solutions are great for not so many files (less than ~60). If they handle more files, they just quit into nothing with no errors. This is because they open too many streams.
This solution is inspired by https://gist.github.com/amiantos/16bacc9ed742c91151fcf1a41012445e
It is a working solution, which works well even with many files (+300) and returns a presigned URL to the zip which contains the files.
Main Lambda:
const AWS = require('aws-sdk');
const S3 = new AWS.S3({
apiVersion: '2006-03-01',
signatureVersion: 'v4',
httpOptions: {
timeout: 300000 // 5min Should Match Lambda function timeout
}
});
const archiver = require('archiver');
import stream from 'stream';
const UPLOAD_BUCKET_NAME = "my-s3-bucket";
const URL_EXPIRE_TIME = 5*60;
export async function getZipSignedUrl(event) {
const prefix = `uploads/id123123/}`; //replace this with your S3 prefix
let files = ["12314123.png", "56787567.png"] //replace this with your files
if (files.length == 0) {
console.log("No files to zip");
return result(404, "No pictures to download");
}
console.log("Files to zip: ", files);
try {
files = files.map(file => {
return {
fileName: file,
key: prefix + '/' + file,
type: "file"
};
});
const destinationKey = prefix + '/' + 'uploads.zip'
console.log("files: ", files);
console.log("destinationKey: ", destinationKey);
await streamToZipInS3(files, destinationKey);
const presignedUrl = await getSignedUrl(UPLOAD_BUCKET_NAME, destinationKey, URL_EXPIRE_TIME, "uploads.zip");
console.log("presignedUrl: ", presignedUrl);
if (!presignedUrl) {
return result(500, null);
}
return result(200, presignedUrl);
}
catch(error) {
console.error(`Error: ${error}`);
return result(500, null);
}
}
Helper functions:
export function result(code, message) {
return {
statusCode: code,
body: JSON.stringify(
{
message: message
}
)
}
}
export async function streamToZipInS3(files, destinationKey) {
await new Promise(async (resolve, reject) => {
var zipStream = streamTo(UPLOAD_BUCKET_NAME, destinationKey, resolve);
zipStream.on("error", reject);
var archive = archiver("zip");
archive.on("error", err => {
throw new Error(err);
});
archive.pipe(zipStream);
for (const file of files) {
if (file["type"] == "file") {
archive.append(getStream(UPLOAD_BUCKET_NAME, file["key"]), {
name: file["fileName"]
});
}
}
archive.finalize();
})
.catch(err => {
console.log(err);
throw new Error(err);
});
}
function streamTo(bucket, key, resolve) {
var passthrough = new stream.PassThrough();
S3.upload(
{
Bucket: bucket,
Key: key,
Body: passthrough,
ContentType: "application/zip",
ServerSideEncryption: "AES256"
},
(err, data) => {
if (err) {
console.error('Error while uploading zip')
throw new Error(err);
reject(err)
return
}
console.log('Zip uploaded')
resolve()
}
).on("httpUploadProgress", progress => {
console.log(progress)
});
return passthrough;
}
function getStream(bucket, key) {
let streamCreated = false;
const passThroughStream = new stream.PassThrough();
passThroughStream.on("newListener", event => {
if (!streamCreated && event == "data") {
const s3Stream = S3
.getObject({ Bucket: bucket, Key: key })
.createReadStream();
s3Stream
.on("error", err => passThroughStream.emit("error", err))
.pipe(passThroughStream);
streamCreated = true;
}
});
return passThroughStream;
}
export async function getSignedUrl(bucket: string, key: string, expires: number, downloadFilename?: string): Promise<string> {
const exists = await objectExists(bucket, key);
if (!exists) {
console.info(`Object ${bucket}/${key} does not exists`);
return null
}
let params = {
Bucket: bucket,
Key: key,
Expires: expires,
};
if (downloadFilename) {
params['ResponseContentDisposition'] = `inline; filename="${encodeURIComponent(downloadFilename)}"`;
}
try {
const url = s3.getSignedUrl('getObject', params);
return url;
} catch (err) {
console.error(`Unable to get URL for ${bucket}/${key}`, err);
return null;
}
};
Using streams may be tricky as I'm not sure how you could pipe multiple streams into an object. I've done this several times using standard file object. It's a multistep process and it's quite fast. Remember that Lambda operates in Linux so you have all Linux resources at hand including the system /tmp directory.
Create a sub-directory in /tmp call "transient" or whatever works for you
Use s3.getObject() and write file objects to /tmp/transient
Use the GLOB package to generate an array[] of paths from /tmp/transient
Loop the array and zip.addLocalFile(array[i]);
zip.writeZip('tmp/files.zip');
I've used a similar approach, but I'm facing the issue that some of the files in the generated ZIP file don't have the correct size (and corresponding data). Is there any limitation on the size of the files this code can manage? In my case I'm zipping large files (a few larger than 1GB) and the overall amount of data may reach 10GB.
I do not get any error/warning message, so it seems it all works fine.
Any idea what may be hapenning?

Resources