Retrieve the attributes from a file using nodejs? - node.js

Hi I'm looking for an nodejs code which would probably return the attributes of each file in a folder. I developed the code to retrieve all the file name in a folder and another code to list data's of filename provide by us. But actually I need to return all the files names in a folder with its column name. I'm new to nodejs so someone help me please.
LISTING DATA CODE:
const AWS = require('aws-sdk');
const neatCsv = require('neat-csv');
var s3 = new AWS.S3({});
exports.handler = (event,context,callback)=>{
const params = {
Bucket:'ml-framework-api',
Key: wavicle.csv
};
s3.getObject(params,async(err, result) => {
if (err){
return console.error(err);
}
neatCsv(result.Body).then((parsedData) => {
callback(null,parsedData);
})
})
}
LISTING FILE IN S3:
const AWS = require('aws-sdk')
const s3 = new AWS.S3({
accessKeyId:'-------------',
secretAccessKey:'-------------------',
region:'ap-south-1'
})
const params = {
Bucket:'wavicle'
}
s3.listObjects(params,(err,data)=>{
if(err){
return console.log(err)
}
console.log(data)
})

It's best to start with node's file system api documentation.
Here is a simple example of how to get information about files of a folder (there are many ways, this is quickly from the example in the documentation above):
const fsp = require("fs/promises");
async function dirFilesInfo(path) {
const dir = await fsp.opendir(path);
for await (const dirEntry of dir) {
const fileInfo = await fsp.stat("./" + dirEntry.name);
console.log(dirEntry.name, fileInfo);
}
}
dirFilesInfo("./").catch(console.error);

Related

Upload static web to AWS s3 with Node JS break static web hosting

I am planning to deploy next.js static export to s3 bucket with Node JS script.
I have setup a S3 bucket for static website hosting.
I get the expected behaviour when I simply drag and drop the static export to the S3 bucket so I pretty sure I setup the S3 bucket correctly.
But when I try to upload it with a Node JS script, despite all the files appear in the bucket, the behaviour of static website hosting seems to break.
My script is basically copy from with a twist on reading the environment variable on .env:
Upload entire directory tree to S3 using AWS sdk in node js
More Info and steps to reproduce this problem is in the Testing Repo.
Testing Repo:
https://github.com/vmia159/aws-upload-test
I appreciate if someone can have an idea about the issues.
https://github.com/aws/aws-sdk-js/issues/4279
As chrisradek points out, you need to provide content type to make it work
require('dotenv').config();
const bucketName = process.env.BUCKET_NAME;
const { promises: fs, createReadStream } = require('fs');
const path = require('path');
const { S3 } = require('aws-sdk');
const mime = require('mime-types');
const s3 = new S3({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
});
const uploadDir = async (s3Path, bucketName) => {
// Recursive getFiles from
// https://stackoverflow.com/a/45130990/831465
async function getFiles(dir) {
const dirents = await fs.readdir(dir, { withFileTypes: true });
const files = await Promise.all(
dirents.map(dirent => {
const res = path.resolve(dir, dirent.name);
return dirent.isDirectory() ? getFiles(res) : res;
})
);
return Array.prototype.concat(...files);
}
const files = await getFiles(s3Path);
const uploads = files.map(filePath =>
s3
.putObject({
Key: path.relative(s3Path, filePath),
Bucket: bucketName,
Body: createReadStream(filePath),
ContentType: mime.lookup(filePath)
})
.promise()
.catch(err => {
console.log(`fail to upload ${filePath}`);
})
);
return Promise.all(uploads);
};
const uploadProcess = async () => {
await uploadDir(path.resolve('./out'), bucketName);
console.log('Upload finish');
};
uploadProcess();

upload to s3 with fileS buffer

I am trying to upload to s3 with bulk files.
Somehow if I am uploading with callback, it'll work properly but I want to push all the return data into an array then do something after. But it doesn't work.
I looked online, I was saw answers such as using async await or recurssive would work but still it's not working though. I even tried using reduce but no luck too
example of my reduce
return files.reduce((accumulator, current) => {
const {path, buffer} = current;
const s3 = new AWS.S3();
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
s3.putObject(awsS3sdkParams(path, buffer), function ( err, data ) {
const { protocol, host } = this.request.httpRequest.endpoint;
data.params = this.request.params;
data.params.url = `${protocol}//${host}/${data.params.Key}`;
return [...accumulator, data];
});
}, []);
example using recurrsive
const result = [];
const helper = (files) => {
const {path, buffer} = files[0];
const s3 = new AWS.S3();
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
s3.putObject(UploadService.awsS3sdkParams(path, buffer), function (err, data){
const { protocol, host } = this.request.httpRequest.endpoint;
data.params = this.request.params;
data.params.url = `${protocol}//${host}/${data.params.Key}`;
UtilsService.clDebug(data, 'data');
result.push(data);
files.shift();
if(files.length > 0) return helper(files);
});
};
helper(files);
return results;
example using promise
const result = [];
for(let {path, buffer} of files){
const s3 = new AWS.S3();
s3.putObject(awsS3sdkParams(path, buffer)).promise()
.then(file => {
result.push(file);
})
.catch(err => {
console.log(err, 'errs');
});
}
I can pretty much understand why result is always [] but how can I make it work though?
Reason why I cannot use async await is because I tried but then somehow after files are either uploaded with bad data that I cannot even open the file, or keys would be the same...
Does anyone has any other suggestions or advice?
Thanks in advance for any

How to read a txt file, stored in a google storage bucket, from a cloud function, in node

I've written some node.js code which is sitting in the source of a Cloud Function. When it runs I want to read a text file from a google storage bucket, and process it.
The Code runs fine when running locally, but for some reason doesn't work when running in the Cloud Function. I would expect something written out from the console logs.
I can't see any errors, as I thought it might be a permissions problem (might be looking in the wrong place though).
Any ideas?
The awaits and async's were just because I wanted it to wait for the response before continuing, but that seems to have no effect on it either.
const fileName = 'testData.txt';
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
const bucket = storage.bucket('my_bucket_name');
const remoteFile = bucket.file(fileName);
await remoteFile.download(async function(err, contents) {
console.log("file err: "+err);
console.log("file data: "+contents);
});
What you can do is to verify that the runtime account for the function has the necessary permissions to access the bucket. In general the runtime account is PROJECT_ID#appspot.gserviceaccount.com and add at least the Storage Object Viewer (you can check more roles here).
Then, test the function again. If something goes wrong, please check the logs of the function.
EDIT
Not sure, but maybe seems to be something with the code. I've used the following to test the function and works perfect:
index.js:
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
const bucket = storage.bucket('bucket_name');
const fileName = 'test.txt';
const remoteFile = bucket.file(fileName);
exports.helloWorld = (req, res) => {
console.log('Reading File');
var archivo = remoteFile.createReadStream();
console.log('Concat Data');
var buf = '';
archivo.on('data', function(d) {
buf += d;
}).on('end', function() {
console.log(buf);
console.log("End");
res.send(buf);
});
};
package.json:
{
"name": "sample-http",
"version": "0.0.1",
"dependencies": {
"#google-cloud/storage": "^4.7.0"
}
}
async function readStorageFile(obj){
try{
obj.Result = ''
if (obj.filename===undefined) return
bucket = gcs.bucket('gs:yourinfo');
//Get File
await bucket.file(obj.filename).download()
.then(async data => {
obj.data = data
obj.Result = 'OK'
return
})
.catch(err => {
return
})
return
}
catch (err){
return
}
}
obj = {filename:'TEST1.json'}
await readStorageFile(obj,'testing')
if (obj.Result==='OK') { console.log('obj.data='+obj.data) }
else { console.log('Not Found')}
return

minizip-asm extract function takes forever to execute

I am trying to fetch an AES encrypted password protected zip file from a google storage and extract a csv file from it. I am using google cloud functions for it with node 6.
I've tried using minizip-asm.js library to extract the file. It works intermittently. I am a newbie when it comes to node so would really appreciate some help :).
Here's the relevant piece of code which might help. Could someone help me figure out what's going wrong here.
exports.processFile = (event, callback) => {
const file = event.data;
const filename = file.name;
const projectId = "abc1234";
const bucketName = "abc_reports";
const Storage = require('#google-cloud/storage');
const storage = Storage({
projectId: projectId
});
const folder = storage.bucket(bucketName);
const minizip = require('minizip-asm.js');
if (file.metageneration === '1' && filename.match(".zip") != null) {
// metageneration attribute is updated on metadata changes.
// on create value is 1
console.log(`File ${file.name} uploaded.`);
folder.file(filename).download().then(function(data) {
console.log('Download of file complete');
//create csv file
var csvName = filename.split(".zip")[0] + ".csv"
var mz = new minizip(data[0]);
console.log(data[0]);
console.log(mz.list());
var extract = mz.extract(mz.list()[0].filepath,{
password: 'ABC#123'
})
console.log("extracted");
//write unzipped contents to file
folder.file(csvName).save(extract, function(err) {
if (!err) {
console.log("unzipped csv");
}
else console.log("Error in saving csv : "+err);
});
});
});
}
callback(null, 'Success!');
};
Thanks for the help.

Load and parse an yaml file from S3, using a nodejs AWS lambda

I have an AWS lambda that has to...
read from S3 an yaml file,
transform the content into an object,
than do some wonderful staff with it.
I can handle the last point, but I do not know how to read and parse that yaml file.
Here is the code I need to complete:
const AWS = require('aws-sdk');
const YAML = require('js-yaml');
const S3 = new AWS.S3();
module.exports.handler = (event, context, callback) => {
const [record] = event.Records;
const bucket = record.s3.bucket.name;
const { key } = record.s3.object;
const params = { Bucket: bucket, Key: key };
console.log(params);
S3.getObject(params).promise()
.then((x) => {
// ?????????????????????????????????????????
// HOW TO DO SOME MAGIC AND GET THE OBJECT ?
})
.then((fileContentObject) => {
// DO SOME WONDERFUL STAFF (example: console.log :) )
console.log(JSON.stringify(fileContentObject))
})
.then(() => callback)
.catch(callback);
};
Feel free to suggest another approach to read and parse the yaml file. I prefer a Promise approach if possible.
I've finally solved the problem. "It was easy", of course!
Here is the code in lambda:
S3.getObject(params).promise()
.then((configYaml) => {
// Get the content of the file as a string
// It works on any text file, should your config is in another format
const data = configYaml.Body.toString('utf-8');
console.log(data);
// Parse the string, which is a yaml in this case
// Please note that `YAML.parse()` is deprecated
return YAML.load(data);
})
.then((config) => {
// Do some wonderful staff with the config object
console.log(`• Config: ${JSON.stringify(config)}`);
return null;
})
.then(callback)
.catch(callback);
All I was asking for is: YAML.load(configYaml.Body.toString('utf-8')).

Resources