I am completely new to node js .
I am trying to code below steps:
Download a file from AWS S3 folder
Then upload it to some other AWS s3 folder.
So I have searched online and created similar code in node js .
The below code is for the same .
What I see here is the downloadFile and uploadFile functions run in parallel and uploadFile runs first, it seems.
How to run them in sequence?
const aws = require('aws-sdk');
var s3 = new aws.S3();
var fs = require('fs');
// TODO implement
var params = { Bucket: "buckets3", Key: "input_pdf_img/Gas_bill_sample.pdf" };
const filename = 'Gas_bill_sample.pdf';
const bucketName = "translation-bucket-qa-v1";
const key = "input_pdf_img/Gas_bill_sample.pdf";
const key2 = "output_pdf2docx_img/"+filename;
//console.log(filename);
const tmp_filename = "/tmp/Gas_bill_sample.pdf";
console.log(filename);
const downloadFile = (tmp_filename, bucketName, key) => {
const params2 = {
Bucket: bucketName,
Key: key
};
s3.getObject(params, (err, data) => {
if (err) console.error(err);
fs.writeFileSync(tmp_filename, data.Body.toString());
//console.log(`${filePath} has been created!`);
});
};
//downloadFile(tmp_filename, bucketName, key);
//console.log('download done');
//await sleep(1000);
//upload
const uploadFile = (tmp_filename) => {
// Read content from the file
const fileContent = fs.readFileSync(tmp_filename);
// Setting up S3 upload parameters
const params2 = {
Bucket: bucketName,
Key: key2, // File name you want to save as in S3
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params2, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
downloadFile(tmp_filename, bucketName, key);
console.log('download done');
//setTimeout(() => {console.log("Let the download finish")}, 6000);
uploadFile(tmp_filename);
//setTimeout(() => {console.log("Let the download finish")}, 6000);const aws = require('aws-sdk');
var s3 = new aws.S3();
var fs = require('fs');
// TODO implement
var params = { Bucket: "buckets3", Key: "input_pdf_img/Gas_bill_sample.pdf" };
const filename = 'Gas_bill_sample.pdf';
const bucketName = "translation-bucket-qa-v1";
const key = "input_pdf_img/Gas_bill_sample.pdf";
const key2 = "output_pdf2docx_img/"+filename;
//console.log(filename);
const tmp_filename = "/tmp/Gas_bill_sample.pdf";
console.log(filename);
const downloadFile = (tmp_filename, bucketName, key) => {
const params2 = {
Bucket: bucketName,
Key: key
};
s3.getObject(params, (err, data) => {
if (err) console.error(err);
fs.writeFileSync(tmp_filename, data.Body.toString());
//console.log(`${filePath} has been created!`);
});
};
//downloadFile(tmp_filename, bucketName, key);
//console.log('download done');
//await sleep(1000);
//upload
const uploadFile = (tmp_filename) => {
// Read content from the file
const fileContent = fs.readFileSync(tmp_filename);
// Setting up S3 upload parameters
const params2 = {
Bucket: bucketName,
Key: key2, // File name you want to save as in S3
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params2, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
downloadFile(tmp_filename, bucketName, key);
console.log('download done');
//setTimeout(() => {console.log("Let the download finish")}, 6000);
uploadFile(tmp_filename);
//setTimeout(() => {console.log("Let the download finish")}, 6000);
Tried time out and other ways but no help.
Since the const runs in parallel error is "No such file or directory" as the download file runs after uploadFile.
Related
Is it possible to store ffmpeg output directly to s3 without downloading it in local or any other storage?
Below is my understanding of ffmpeg which converts format of video. I have done conversion part but i need to store it's output directly to s3 bucket so anyone have idea regarding this problem ?
const AWS = require('aws-sdk');
const fs = require('fs');
const ffmpeg = require('fluent-ffmpeg');
const axios = require('axios');
const s3 = new AWS.S3({
endpoint: 's3-ap-south-1.amazonaws.com', // Put you region
accessKeyId: S3_ACCESS_KEY_ID, // Put you accessKeyId
secretAccessKey: S3_ACCESS_SECRET_KEY, // Put you accessKeyId
Bucket: S3_BUCKET_NAME, // Put your bucket name
signatureVersion: 'v4',
region: 'ap-south-1' // Put you region
});
var params = {
Bucket: S3_BUCKET_NAME,
Delimiter: '',
Prefix: S3_STORE_PATH
};
s3.listObjects(params, function (err, data) {
if (err) throw err;
console.log(data);
data.Contents.forEach(function (obj, index) {
const file_name = obj.Key;
const type = "mp4";
console.log(obj.Key)
const url = s3.getSignedUrl('getObject', {
Bucket: S3_BUCKET_NAME,
Key: obj.Key,
Expires: signedUrlExpireSeconds
});
console.log("SIGNED URL= ", url);
const filename = file_name.split('.').slice(0, -1).join('.');
const localFileOutput = `${filename}.${type}`;
// const localFileOutput = `${bucket_url}${filename}.${type}`;
console.log(localFileOutput);
const key = `${filename}.${type}`;
const convert_video = async (req,res) => {
await new Promise((resolve, reject) => {
ffmpeg().input(url)
.toFormat('mp4')
.output(localFileOutput)
.on('end', async () => {
const params = {
Bucket: S3_BUCKET_NAME,
Key: key,
Body: localFileOutput
}
// const fileContent = await fs.readFileSync(localFileOutput);
await s3.putObject(params).promise();
resolve();
}).run();
});
// res.send("success")
}
convert_video();
});
});
I am struggling with unzipping the contents in AWS S3. AWS S3 does not provide the functionality of unzipping the zip folder in the S3 bucket directly. I facing one error . upload code screenshot attached.
"Error: Invalid CEN header (bad signature)"
Any advice or guidance would be greatly appreciated.
My node Js code to upload the zip file:
const AWS = require('aws-sdk');
const s3 = new AWS.S3({signatureVersion: 'v4'});
exports.handler = async (event,context) => {
const bucket = 'bucket-name';
console.log(event)
const body = event.body;
const key=JSON.parse(body).key
console.log(key)
const params = {
Bucket: bucket,
Key: key,
ContentType: 'application/zip',
Expires: 60
};
try{
const signedURL = await s3.getSignedUrl('putObject', params);
const response = {
err:{},
body:"url send",
url:signedURL
};
return response;
}catch(e){
const response = {
err:e.message,
body:"error occured"
};
return response;
}};
My NodeJs code to extract the zip file:
const S3Unzip = require('s3-unzip');
exports.s3_unzip = function(event, context, callback) {
const filename = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const bucketname = event.Records[0].s3.bucket.name;
console.log(event.Records[0].s3.object.key);
new S3Unzip({
bucket: bucketname,
file: filename,
deleteOnSuccess: true,
verbose: true,
}, function(err, success) {
if (err) {
callback(err);
} else {
callback(null);
}
});
}
I'm trying to write an Lambda function that unzips zip files in one S3 directory and extract into another. I had this working in Python but nobody else in my group likes Python so I'm converting it to Node.js which I'm not very good at.
I'm trying to use the unzipper package and I'm able to get a list of files in the zip file using unzipper.Open.S3, but I can't figure out how to stream the files in the zip file into S3.
The meat of the code looks like
const directory = await unzipper.Open.s3(s3,{Bucket: bucket, Key: zip_file});
directory.files.forEach(file => {
console.log("file name = " + file.path + ", type = " + file.type)
const key = dir[0] + "/output/" + file.path;
const params = { Bucket: bucket, Key: key };
const { writeStream, promise } = uploadStream(params)
file.stream().pipe(writeStream);
promise.then(() => {
console.log('upload completed successfully');
}).catch((err) => {
console.log('upload failed.', err.message);
});
});
const uploadStream = ({ Bucket, Key }) => {
const pass = new stream.PassThrough();
return {
writeStream: pass,
promise: s3.upload({ Bucket, Key, Body: pass }).promise()
};
}
I get the console.log for each file, but neither of the logs in promise.then and .catch comes out and no new files appear in S3.
Never mind, I found this code that works better:
exports.handler = async (event) => {
const params = {
Key: zip_directory + "/" + zip_file,
Bucket: input_bucket
};
const zip = s3
.getObject(params)
.createReadStream()
.pipe(unzipper.Parse({ forceStream: true }));
const promises = [];
let num = 0;
for await (const e of zip) {
const entry = e;
const fileName = entry.path;
const type = entry.type;
if (type === 'File') {
const uploadParams = {
Bucket: output_bucket,
Key: output_directory + fileName,
Body: entry,
};
promises.push(s3.upload(uploadParams).promise());
num++;
} else {
entry.autodrain();
}
}
await Promise.all(promises);
};
I have the following code below, which is a lambda function to get content from a s3Object zip file. I know for a fact that I am not resolving the list of promises and need a little direction on how to resolve. I have read several codes on here but having a hard time applying it to my code. Any assistance would be greatly appreciated.
// dependencies
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
var zip = JSZip.loadAsync(data.Body).then(function (content){
return content;
});
zip.then(function(result){
var entries = Object.keys(result.files).map(function (name) {
if(name.indexOf("TestStatus") != -1){
return name;
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map(function(entry) {
return result.file(entry).async("text").then(function(fileContent){
return fileContent;
});
});
Promise.all(listOfPromises).then((values) =>{
values.forEach(function(value){
console.log(value);
});
});
});
} catch (error) {
context.fail(error);
return;
}
};
Modified/Corrected code
// dependencies
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
var zip = JSZip.loadAsync(data.Body);
return zip.then(function(result){
var entries = Object.keys(result.files).map((name) =>{
if(name.indexOf("TestStatus") != -1){
return result.files[name];
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map((entry) => {
return entry.async("text")
.then((u8) => {
return [entry.name, u8];
}).catch(error => console.error(error));
});
var promiseOfList = Promise.all(listOfPromises);
promiseOfList.then(function (list) {
console.log(list.toString());
});
});
} catch (error) {
context.fail(error);
return;
}
};
If you look closely you are not retuning anything it is why it stays on Pending
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
// here is the problem
// var zip = JSZip.loadAsync(data.Body).then(function (content){
// return content;
// }
var zip = await JSZip.loadAsync(data.body)
return zip.then(function(result){
var entries = Object.keys(result.files).map(function (name) {
if(name.indexOf("TestStatus") != -1){
return name;
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map(function(entry) {
return result.file(entry).async("text").then(function(fileContent){
return fileContent;
});
});
console.log("Helo");
Promise.all(listOfPromises).then((values) =>{
values.forEach(function(value){
console.log(value);
});
});
});
} catch (error) {
context.fail(error);
return;
}
};
```
I have a file in the path ./files/myfile.zip
That I would like to upload to s3.
I have a function:
const writeS3Async = (bucketName, fileName, strInput) => {
const params = {
Bucket: bucketName,
Key: fileName,
Body: strInput,
};
return s3.upload(params).promise();
};
The writeS3Async method is used for uploading strings into an s3 object, not files.
Interestingly, I could not find a decent piece of code for a direct file upload.
You need to read the file first. Something like this
var AWS = require('aws-sdk'),
fs = require('fs');
fs.readFile('./files/myfile.zip', function (err, data) {
if (err) { throw err; }
var s3 = new AWS.S3();
const params = {
Bucket: bucketName,
Key: fileName,
Body: data
};
return s3.client.putObject(params).promise();
});