I am working with nodeJS and Azure functions. I am trying to get the content of a blob (pptx) and then further work with that pptx (unzip it with admzip).
However, whenever I try to get the content, the function just stops without any error and after some time it times out. I tried getting the properties of the blob first (to check if the blob exists) and that works.
Here is my function:
const storage = require('azure-storage');
const STORAGE_ACCOUNT_NAME = 'storage-account';
const ACCOUNT_ACCESS_KEY = 'storage-key';
let AdmZip = require('adm-zip');
let fs = require('file-system');
const blobService = storage.createBlobService(STORAGE_ACCOUNT_NAME, ACCOUNT_ACCESS_KEY);
module.exports = function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
getBlobProperties('default-powerpoint', 'download.pptx').then((properties) => {
context.log('Properties: ', properties);
getBlobContent('default-powerpoint', 'download.pptx').then((content) => {
context.log('Blob Content: ', content);
})
});
};
function getBlobProperties(containerName, fileName) {
return new Promise((resolve, reject) => {
blobService.getBlobProperties(
containerName,
fileName,
function (err, properties, status) {
if (err) {
reject(err);
} else {
resolve(properties);
}
});
})
}
function getBlobContentAsStream(containerName, fileName, res) {
return new Promise((resolve, reject) => {
blobService.getBlobToStream(containerName, fileName, res, function (err, results) {
if (err) {
reject(err);
} else {
resolve(JSON.stringify(results, null, 2));
}
});
})
}
function getBlobContent(containerName, blobName) {
return new Promise((resolve, reject) => {
blobService.getBlobToText(
containerName,
blobName,
function (err, blobContent, blob) {
if (err) {
reject(err);
} else {
resolve({
'content': blobContent,
'blob': blob
});
}
});
})
}
As you can see I tried both getBlobToStream and getBlobToText but with the same result. The getBlobProperties works fine and I get all the information about the blob, just not the content.
Can anyone please help me get the content of the blob.
Edit:
This is the output of the properties if anyone is interested:
BlobResult {
container: 'default-powerpoint',
name: 'download.pptx',
metadata: {},
lastModified: 'Wed, 14 Aug 2019 08:28:16 GMT',
creationTime: 'Wed, 14 Aug 2019 08:28:16 GMT',
etag: '"something"',
blobType: 'BlockBlob',
contentLength: '4658',
serverEncrypted: 'true',
requestId: 'someID',
contentSettings: { contentType: 'image/jpeg' },
lease: { status: 'unlocked', state: 'available' },
copy:
{ id: 'id123',
status: 'success',
source: 'sourceURL',
progress: '4658/4658',
bytesCopied: 4658,
totalBytes: 4658,
completionTime: 'Wed, 14 Aug 2019 08:28:16 GMT' } }
Here is the working code i have used in my app
return new Promise(async r => {
bst.getBlobToText(containername, name, (err, text) => r(err ? null : text));
})
Full SourceCode
I was able to get solve this to work using code
const storage = require('azure-storage');
const fs = require('fs');
const STORAGE_ACCOUNT_NAME = '<account_name>';
const ACCOUNT_ACCESS_KEY = '<access_key>';
const blobService = storage.createBlobService(STORAGE_ACCOUNT_NAME, ACCOUNT_ACCESS_KEY);
function getBlobContentAsStream(containerName, fileName, res) {
return new Promise((resolve, reject) => {
blobService.getBlobToStream(containerName, fileName, fs.createWriteStream('task1-download.txt'), function(error, serverBlob) {
if(!error) {
resolve(serverBlob);
} else {
reject(err);
}
});
})
}
module.exports = function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
context.log('Starting...');
getBlobContentAsStream('default-powerpoint', 'download.pptx').then((content) => {
context.log('Blob Content: ', content);
context.done();
}, function(err) {
console.log.error(err);
context.done();
});
};
And trace output
The problem that happens in your code you can actually see in Application Insights trace if you connected Functions to it. You didn't get any errors because you didn't add error handling for your then callback executor.
getBlobContent('default-powerpoint', 'download.pptx').then((content) => {
context.log('Blob Content: ', content);
context.done();
})
Use
getBlobContent('default-powerpoint', 'download.pptx').then((content) => {
context.log('Blob Content: ', content);
context.done();
}, function(err) {
console.log(err);
context.done();
}))
You would see
With details
Error: An incorrect number of bytes was read from the connection.
The connection may have been closed.
So the problem you are having with getBlobToText is that it tries to return Buffer object as string and it fails to validate MD5. I read somewhere it's possible to using write to stream function to write to buffer instead of file but I can't find it right now.
I would probably grab some NodeJS memory stream library and try output it there as I would assume you don't want to save directly to file. But maybe you do, decide yourself.
If you would end up using 'fs' library remember to use recommended safe non blocking patterns like this
const fs = require('fs');
const util = require('util');
const readFileAsync = util.promisify(fs.readFile);
module.exports = async function (context) {
try {
const data = await readFileAsync('./hello.txt');
} catch (err) {
context.log.error('ERROR', err);
// This rethrown exception will be handled by the Functions Runtime and will only fail the individual invocation
throw err;
}
context.log(`Data from file: ${data}`);
}
Might be the issue is that the api is changed. I just checked below, callback function takes only two arguments in getBlobToText:
https://github.com/Azure-Samples/storage-blobs-node-quickstart/blob/master/index.js
const downloadBlob = async (containerName, blobName) => {
const dowloadFilePath = path.resolve('./' + blobName.replace('.txt', '.downloaded.txt'));
return new Promise((resolve, reject) => {
blobService.getBlobToText(containerName, blobName, (err, data) => {
if (err) {
reject(err);
} else {
resolve({ message: `Blob downloaded "${data}"`, text: data });
}
});
});
};
Related
I read Pipe a stream to s3.upload()
but im having difficulty with I am not sure if that actually solves and I have tried.
What I am doing is a get call to www.example.com. this returns a stream, I want to upload that stream to s3.
heres my try.
fetch('https://www.example.com',fileName{
method: 'GET',
headers: {
'Authorization': "Bearer " + myAccessToken,
},
})
.then(function(response) {
return response.text();
})
.then(function(data) {
uploadToS3(data)
});
const uploadToS3 = (data) => {
// Setting up S3 upload parameters
const params = {
Bucket:myBucket,
Key: "fileName",
Body: data
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
output: ///File uploaded successfully. https://exampleBucket.s3.amazonaws.com/fileName.pdf
however this is blank.
I figured it out, but i did not keep using fetch.
and I actually download the file, then upload it. then delete the file.
function getNewFilesFromExampleDotCom(myAccessToken, fileName, fileKey) {
let url2 = 'https://example.com' + fileKey;
axios
.get(url2, {
headers: { 'Authorization': "Bearer " + myAccessToken },
responseType: 'stream',
})
.then(response => {
let file = fileName;
response.data.pipe(fs.createWriteStream(file))
let myFileInfo = [];
if( myFileInfo.length > 0){
myFileInfo.splice(0, myFileInfo.length)
}
myFileInfo.push(file)
processArray(myFileInfo)
console.log(file + " saved")
})
.catch(error => console.log(error));
}
async function processArray(array) {
for (const item of array) {
await delayedLog(item);
}
console.log('Downloaded!');
console.log('Uploading to s3!');
}
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function delayedLog(item) {
await delay();
uploadFiles(item)
}
async function uploadFiles(file){
uploadToS3List(file)
await new Promise((resolve, reject) => setTimeout(resolve, 1000));
deleteMyFiles(file)
}
const uploadToS3List = (fileName) => {
// Read content from the file
const fileContent = fs.readFileSync(fileName);
// Setting up S3 upload parameters
const params = {
Bucket:"myBucketName",
Key: fileName,
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
function deleteMyFiles(path){
fs.unlink(path, (err) => {
console.log(path + " has been deleted")
if (err) {
console.error(err)
return
}
})
}
I'm experimenting with Node.js in AWS Lambda. And, I've run into a problem with the code below. Result value and error value are always returned blank. I'm pretty sure this is just a scope issue I'm to tired to see. How do I capture the return value and pass it back in the response? I'm sure the programs is connecting to redis because I get an error message if I change the port or URL and I don't when they're set properly.
The return code:
{
"statusCode": 200,
"body": "{\"key\":\"q1\"}"
}
The program code:
const Redis = require("ioredis");
const redis = new Redis(6379, 'fredflenstone.lhpxwy.az.0002.use2.cache.amazonaws.com');
exports.handler = async(event)=>{
let key=event.key;
let response;
let resultValue;
let errorValue;
redis.get(key, (err, result) => {
if (err) {
errorValue=err;
} else {
resultValue=result;
}
});
response={
key: key,
resultValue: resultValue,
errorValue: errorValue
};
return {
statusCode: 200,
body: JSON.stringify(response)
};
};
The problem is due to promises. Your handler execution is completing before redis is returning the result. Following snippet should work:
const Redis = require("ioredis");
const redis = new Redis(6379, 'fredflenstone.lhpxwy.az.0002.use2.cache.amazonaws.com');
exports.handler = async(event)=>{
let key=event.key;
let response;
let resultValue;
let errorValue;
try{
resultValue = await redis.get(key);
}catch(error) {
errorValue = error;
}
response={
key: key,
resultValue: resultValue,
errorValue: errorValue
};
return {
statusCode: 200,
body: JSON.stringify(response)
};
};
It is because your call to "redis.get" is not resolved when "response" is sent.
You need to wait for the response :
await new Promise((resolve) => {
redis.get(key, (err, result) => {
if (err) {
errorValue=err;
} else {
resultValue=result;
}
resolve();
});
})
or even better turn the redis response into a promise response :
await new Promise((resolve, reject) => {
redis.get(key, (err, result) => {
if (err) {
reject(err);
} else {
resolve(result);
}
})
})
.then((result) => resultValue = result)
.catch((err) => errorValue = err)
I am using aws sdk to uplod user input image and then get the image link from aws and i will store the link in mongoDB. In that case when i run .upload() it is async.
const imgSRC = [];
for (let img of image) {
console.log(img);
const params = {
Bucket: process.env.AWS_BUCKET,
Key: `${img.originalname}_${userID}`,
Body: img.buffer,
};
s3.upload(params, (error, data) => {
if (error) {
console.log(error);
res.status(500).json({ msg: "server error" });
}
imgSRC.push(data.Location);
console.log(imgSRC);
});
}
const newPost = new Post({
userID: userID,
contentID: contentID,
posts: [
{
caption: caption,
data: imgSRC,
},
],
});
const post = await newPost.save();
in that case when the .save to mongodb run, there is no imgLinks from aws yet. How can i fix that things.
I've already tried async and it didn't work
You need to use Promise.all() in this manner
const uploadImage = (obj) => {
return new Promise((resolve, reject) => {
const params = {
Bucket: process.env.AWS_BUCKET,
Key: obj.key,
Body: obj.body,
}
s3.upload(params, (error, data) => {
if (error) {
console.log(error);
return reject(error);
}
return data;
});
})
}
const mainFunction = async () => {
const promises = [];
for (let img of image) {
const options = {
key: `${img.originalname}_${userID}`,
body: img.buffer
};
promises.push(uploadImage(options));
}
const result = await Promise.all(promises);
const imgSRC = result.map((r) => { return r.Location });
return imgSRC;
}
If you use await on s3.upload method you should remove the callback for this method.
try {
const data = await s3.upload(params);
imgSRC.push(data.Location);
console.log(imgSRC);
} catch(e) {
console.log(error);
res.status(500).json({ msg: "server error" });
}
Let me know if it works.
I have json file uploaded to s3
then I wrote the following code to Query this file
const aws = require('aws-sdk');
const s3 = new aws.S3();
const bucket = 'hotels.mserver.online';
const objectKey = 'hotelsrates.json';
exports.handler = (event,context,callback) => {
// TODO implement
const response = getS3Objects(bucket,objectKey); //s3.listObjectsV2({}).promise();
console.log(response);
};
function getS3Objects(bucket,key) {
return s3.getObject({ Bucket:bucket, Key:key, ResponseContentType:'application/json '})
.promise().then(file => { return file })
.catch(error => { return error });
}`
but the result is getting null .
I understand what you are trying to accomplish here but that is not the right way to do it.
function getS3Objects(bucket,key){
return s3.getObject({Bucket:bucket,Key:key,ResponseContentType:'application/json'})
.promise().then(file=>{return file})
.catch(error =>{return error});
}`
The part above will still return a promise object, which means that you need to handle it accordingly. Instead of const response = getS3Objects(bucket,objectKey); you want to do
getS3Objects(bucket,objectKey).then(response => console.log(response));
Inside of your handler function.
Furthermore, your usage of s3.getObject function is incorrect. Where first argument is an object - parameters, and the second argument is a callback function.
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data);
Therefore in your case, you want to modify your getS3Objects function a bit. If you want to use promises, then you can do it like this.
function getS3Objects(bucket, key) {
return new Promise((resolve, reject) => {
s3.getObject(
{
Bucket: bucket,
Key: key,
ResponseContentType: 'application/json'
},
(err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
}
);
});
}
Another way that you can do this is as follows:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
async function readFile(Bucket, Key) {
const params = {
Bucket,
Key,
ResponseContentType: 'application/json',
};
const f = await s3.getObject(params).promise();
return f.Body.toString('utf-8');
}
readFile('mybucket', 'xyz.json').then(console.log);
I'm using AVA framework (https://github.com/avajs/ava/blob/HEAD/docs/01-writing-tests.md)
to test Cloud Function on GCP written in NodeJS.
i'm trying to unit test an inner function inside my cloud function.
My testing code looks as follow:
const test = require(`ava`);
const uuid = require(`uuid`);
const sinon = require(`sinon`);
const triggerResultsService = require(`..`).triggerResultsService;
const consoleLog = sinon.stub(console, 'log');
const sendToTransferService = require(`..`).triggerResultsService.sendToTransferService;
test('resolves with unicorn', t => {
const filename = uuid.v4();
sendToTransferService(filename, () =>{
});
return Promise().then(result => {
t.is(result, 'unicorn');
});
});
My Cloud function code looks as the following:
/**
* Generic background Cloud Function to be triggered by Cloud Storage.
*
* #param {object} event The Cloud Functions event.
* #param {function} callback The callback function.
*/
var request = require("request");
exports.triggerResultsService = (event, callback) => {
var file = event.data;
var fileName = file.name.substr(file.name.lastIndexOf('/')).toLowerCase().trim();
if(!fileName.includes('temp-') && fileName.includes('.csv.gz')) {
console.log("file name is in correct location, sending options");
sendToTransferService(file);
sendStatusEmail("Transfer File Call successful");
};
callback();
};
function sendToTransferService(file) {
var options = {
method: 'POST',
uri: process.env.TRANSFER_SERVICE_URL,
body: {
fileName: file.name,
bucketName: file.bucket
},
json: true
};
return new Promise(function (resolve, reject) {
request(options, function (err, resp) {
if (err) {
console.log(err);
return reject({err: err});
}
return resolve({responsebody:resp.body});
});
});
}
function sendStatusEmail(statusMessage) {
var options = {
method: 'POST',
uri: process.env.EMAIL_NOTIFICATION_URL,
body: {
from: process.env.EMAIL_FROM,
to: [process.env.SLACK_EMAIL],
cc: [''],
bcc: [''],
subject: process.env.EMAIL_SUBJECT,
body: statusMessage
},
json: true
};
return new Promise(function (resolve, reject) {
request(options, function (err, resp) {
if (err) {
console.log(err);
return reject({err: err});
}
return resolve({responsebody:resp.body});
});
});
}
I'm not able to reach the function sendToTransferService .
Any ideas what should I require/declare
Thanks in advance
I think your easiest path is to turn that function into its own module, and export it from there. It will be accessible to any other code that imports the module, including your Cloud Functions code.