Node.js Firebase Function sending Base64 image to External API - node.js

I’m using Firebase Functions with a Storage trigger in Node.js to send uploaded image data to an external API endpoint where photos are uploaded.
I’m currently taking images uploaded to a bucket in my Firebase storage, converting them to base64 strings, and plug them into my dictionary for the request.
My current issue is that seems like the dictionary is being cut short. I looked at the console logs on the Firebase console and seems like it ends after the base64 variable.
I’m not sure whether this is a bug with the syntax, or with the way I’m using the base64, or with Firebase Functions. If anyone knows what might be going on, please let me know.
const request = require('request-promise');
const gcs = require('#google-cloud/storage')();
const path = require('path');
const os = require('os');
const fs = require('fs');
const firebase = require('firebase');
exports.identifyUpdate = functions.storage.object().onFinalize((object) => {
const fileBucket = object.bucket;
const filePath = object.name;
const contentType = object.contentType;
const fileName = path.basename(filePath);
if(!filePath.substring(0,filePath.indexOf('/')) == 'updates') {
console.log("Triggered by non-update photo")
return null;
}
console.log("Update photo added")
// Create Firebase app (for Realtime Database access)
var config = {
apiKey: "[apikey]",
authDomain: "[PROJECT_ID].firebaseapp.com",
databaseURL: "https://[PROJECT_ID].firebaseio.com",
storageBucket: "[PROJECT_ID].appspot.com",
};
if(!firebase.apps.length) {
firebase.initializeApp(config);
}
// Trace back to Update stored in Realtime Database
const database = firebase.database().ref()
const pendingRef = database.child('pendingUpdates')
console.log(filePath)
const splitPath = filePath.split(path.sep)
const patientID = splitPath[1]
console.log('Patient ID: ' + patientID)
const updateID = splitPath[2]
console.log('Update ID: ' + updateID)
const updateRef = pendingRef.child(patientID).child(updateID)
console.log('Found Update reference')
const photoRef = updateRef.child('photoURLs').child(fileName)
console.log('Photo Reference: ' + photoRef)
// Download and convert image to base64
const bucket = gcs.bucket(fileBucket)
const tempFilePath = path.join(os.tmpdir(), fileName)
const metadata = {
contentType: contentType
};
var base64;
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
console.log('Image downloaded locally to', tempFilePath)
}).then(() => {
base64 = base64_encode(tempFilePath)
console.log("Base 64: " + base64)
}).then(() => {
// Send image data to Kairos
var options = {
method: 'POST',
uri: 'https://api.kairos.com/recognize',
body: {
'image': base64,
'gallery_name': 'gallerytest1'
},
headers: {
'app_id': '[id]',
'app_key': '[key]'
},
json: true
}
return new Promise (() => {
console.log(options)
request(options)
.then(function(repos) {
console.log('API call succeeded');
console.log('Kairos response: ' + repos);
const apiResult = repos['images']['transaction']['subject_id']
console.log("Transaction " + JSON.stringify(apiResult))
})
.catch(function(err) {
console.log('API call failed')
})
});
})
// Delete app instance (to prevent concurrency leaks)
const deleteApp = () => app.delete().catch(() => null);
deleteApp.call
})
function base64_encode(file) {
// read binary data
var bitmap = fs.readFileSync(file);
// convert binary data to base64 encoded string
return new Buffer(bitmap).toString('base64');
}
Image Output:

Related

google cloud function not uploading to bucket but no error in function

I have a NodeJS function that writes several small svg files locally and then is attempting to upload those files to cloud bucket.
in the function log, i am only seeing the message that file written to local disk. now will upload. But there is no file in the bucket and no error logged anywhere. i have made sure the timeout is set to 9 min (max) so i am sure its not timing out. what else shoudl i check?
any pointers will be appreciated.
exports.createQRCode = functions.storage.object().onFinalize(async (object) =>{
const qrcodeMonkeyKey = functions.config().qrcodemonkey.key;
//console.log(`key for qrcode monkey is ${qrcodeMonkeyKey}`);
const fileBucket = object.bucket; // The Storage bucket that contains the file.
const filePath = object.name; // File path in the bucket.
const contentType = object.contentType; // File content type.
const metageneration = object.metageneration; // Number of times metadata has been generated. New objects have a value of 1.
console.log(fileBucket);
console.log(filePath);
if(!filePath.toLowerCase().endsWith('.csv'))
return console.log('not a csv so no need to anything fancy');
const bucket = admin.storage().bucket(fileBucket);
const filePathComps = filePath.split('/');
const folderName = filePathComps[filePathComps.length-3];
if(folderName !== "qrcode")
return console.log('not a qr code csv so no need to anything fancy');
const fileName = filePathComps[filePathComps.length-1];
console.log(fileName);
const path = require('path');
const os = require('os');
const fs = require('fs');
const tempFilePath = path.join(os.tmpdir(), fileName);
const metadata = {
contentType: contentType,
};
await bucket.file(filePath).download({destination: tempFilePath});
const csv = require('csv-parser')
const results = [];
fs.createReadStream(tempFilePath)
.pipe(csv({headers:
['uri','filename','foldername']
,skipLines:1
}))
.on('data', async (data) => {
const x = data;
results.push(data);
//results.push({id:x.id,phoneNumber:x.phoneNumber,isInternational:x.isInternational,message:x.messageText,respStatus:resp.status,responsedata:resp.data});
})
.on('end',async () => {
pArray = [];
results.forEach(x =>{
pArray.push(createQRCodeAndUpload(qrcodeMonkeyKey,x.filename,x.uri,x.foldername));
});
const finaloutput = await Promise.all(pArray);
console.log(JSON.stringify(finaloutput));
return;
});
});
const createQRCodeAndUpload = async (qrcodeMonkeyKey,fileName, url,foldername) =>{
const bucket = admin.storage().bucket('vmallapp.appspot.com');
const path = require('path');
const os = require('os');
const fs = require('fs');
var axios = require("axios").default;
console.log('processing ' + url);
if(url !==""){
const dataToSend = {
data : url,
config :{
body:'circle',
eye:'frame14',
eyeBall:'ball16',
bodyColor:"#032b5c",
bgColor:"#84d4e2",
"logo":"ae600e1267b9e477f0b635b60ffaec1d1c18d93b.png"
},
size:1200,
download:false,
file:'svg',
gradientOnEyes:true
}
var options = {
method: 'POST',
url: 'https://qrcode-monkey.p.rapidapi.com/qr/custom',
headers: {
'content-type': 'application/json',
'x-rapidapi-host': 'qrcode-monkey.p.rapidapi.com',
'x-rapidapi-key': qrcodeMonkeyKey
},
data: dataToSend
};
var response = await axios.request(options);
console.log('qrcode monkey returned status' + response.status);
const outputFilePath = path.join(os.tmpdir(), `${fileName}.svg`);
fs.writeFileSync(outputFilePath, response.data);
console.log(`${fileName}.svg written to local disk. now will upload`);
try{
await bucket.upload(outputFilePath, {
destination: `qrcode/output/${fileName}.svg`
});
}catch(error){
console.log('error in uploding ' + error);
}
console.log('lets delete the file now and clean up local storage');
fs.unlinkSync(outputFilePath);
return 'all done';
}
}

Blank Image on bucket.upload

I am attempting to upload an image to my firebase storage with nodejs. I've used the following code:
const bucketName = 'mybucket.appspot.com';
// The path to your file to upload
const filePath = 'C:/Users/username/OneDrive/Desktop/testfunctions/dogs.png';
// The new ID for your GCS file
const destFileName = 'your-new-file-name.png';
//function upload() {
const {Storage} = require('#google-cloud/storage');
// Creates a client
const storage = new Storage();
async function uploadFile() {
await storage.bucket(bucketName).upload(filePath, {
destination: destFileName,
});
console.log(`${filePath} uploaded to ${bucketName}`);
}
uploadFile().catch(console.error);
However, upon uploading to my storage, the image fails to load, it is an infinite spinning bar, or there is none but just the file name.
Any help is appreciated.
enter image description here
enter image description here
I Have Made This Code A While Ago When Searching I Saw That Nobody Solved Your Problem.
This Method Will Save It In The "Storage". In Addition, Created A Link For It In "Firebase Realtime Database"
//Important Imports
var admin = require("firebase-admin");
var serviceAccount = require("./serviceAccountKey.json");
const { uuid } = require("uuidv4");
const fs = require("fs");
//This Is Important Line To Tell The Firebase URL + STORAGE URL + CONNECT IT TO SERVICE ACCOUNT ADMIN
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL:
"https://yourfirebase.europe-west1.firebasedatabase.app",
storageBucket: "gs://yourfirebasestorage.com",
});
// Get A Database Reference
const db = admin.database();
const ref = db.ref("Parent");
const usersRef = ref.child("Child");
uploadImageToFirebase("imageName", ".png");
//Upload Image To Firebase Function
function uploadImageToFirebase(imageName, imageFormat) {
var myBucketLink = "yourfirebasestoragewithoutgs.appspot.com";
// CHANGE: TO WERE YOU WANT THE ROOT TO BE
//const usersRef = ref.child("anotherPath");
var bucket = admin.storage().bucket();
var filename = "./" + imageName + imageFormat;
try {
if (fs.existsSync(filename)) {
var UUID = uuid();
async function uploadFile() {
const metadata = {
metadata: {
// This Line Is Very Important. It's To Create A Download Token.
firebaseStorageDownloadTokens: UUID,
},
contentType: "image/png",
cacheControl: "public, max-age=31536000",
};
// Uploads A Local File To The Bucket
await bucket.upload(filename, {
// Support For HTTP Requests Made With `Accept-Encoding: gzip`
gzip: true,
metadata: metadata,
});
console.log(`${filename} uploaded.`);
var HTTP_ImageLink =
"https://firebasestorage.googleapis.com/v0/b/" +
myBucketLink +
"/o/" +
imageName +
imageFormat +
"?alt=media&token=" +
UUID;
console.log(HTTP_ImageLink);
usersRef.child("images").set({
link: HTTP_ImageLink,
});
}
uploadFile().catch(console.error);
} else {
console.log("Can't Upload");
}
} catch (err) {
console.log(err);
}
}

Read excel file uploaded to s3 via node lambda function

I am trying to parse through an excel file that is uploaded to s3 using read-excel-file in a node lambda function that triggers on any s3 put. Here is my code which currently doesn't work. Can somebody tell me where I am going wrong?
const aws = require("aws-sdk");
const s3 = new aws.S3({ apiVersion: "2006-03-01" });
const readXlsxFile = require("read-excel-file/node");
exports.handler = async (event, context) => {
// Get the object from the event and show its content type
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(
event.Records[0].s3.object.key.replace(/\+/g, " ")
);
const params = {
Bucket: bucket,
Key: key
};
try {
const doc = await s3.getObject(params);
const parsedDoc = await readXlsxFile(doc);
console.log(parsedDoc)
} catch (err) {
console.log(err);
const message = `Error getting object ${key} from bucket ${bucket}. Make sure they exist and your bucket is in the same region as this function.`;
console.log(message);
throw new Error(message);
}
};
I haven't used lambda functions, but I have done something very similar in firebase functions. I used convert-excel-to-json.
I first downloaded the excel file from firebase storage to the firebase functions machine. Then use this npm module to extract the information.
I don't have time to format the code, but I can leave it here for reference:
// Runs when excel file is uploaded to storage
exports.uploadOrder = functions.storage.object().onFinalize(async (file) => {
const fileBucket = file.bucket;
const filePath = file.name || "null";
const filePathList = filePath?.split("/") || ["null"];
const fileName = path.basename(filePath);
if (filePathList[0] !== "excel_orders") {
return;
}
const uid = filePathList[1];
console.log("User ID: " + uid);
const bucket = admin.storage().bucket(fileBucket);
const tempFilePath = path.join(os.tmpdir(), fileName);
console.log(tempFilePath);
await bucket.file(filePath).download({ destination: tempFilePath });
const result = excelToJson({
sourceFile: tempFilePath,
});
var ordersObj: any[] = result.Sheet1;
ordersObj.shift();
console.log(ordersObj);
var orders: any[] = [];
for (let i = 0; i < ordersObj.length; i++) {
const order: Order = {
package_description: ordersObj[i].A,
package_type: ordersObj[i].B,
country: ordersObj[i].C,
address: ordersObj[i].D,
curstomer_name: ordersObj[i].E,
customer_phone: ordersObj[i].F,
collection_ammount: ordersObj[i].G,
order_date: ordersObj[i].H,
delivery_date: ordersObj[i].I,
delivery_time: ordersObj[i].J,
status: "pending",
assignedTo: "",
merchantID: uid,
};
orders.push(order);
}
});

How to upload images and files to Azure Blob Node.js

I have node.js application with frontend in Angular
I need to upload files and images to Azure blob
I have created container and setup the environment according to MS documentation (https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-nodejs )
the v12 version.
My functions works for creating and uploading the created file to Azure blob, I could not figure how do I upload the posted file from the client to Azure Blob, below is my code in Node.js TypeScript
import * as formidable from 'formidable';
import * as fs from 'fs';
const { BlobServiceClient } = require('#azure/storage-blob');
const uuidv1 = require('uuid/v1');
const dotenv = require('dotenv');
dotenv.config();
class BlobController {
private AZURE_STORAGE_CONNECTION_STRING = process.env.CONSTRINGBlob;
constructor(router) {
router.post('/file', this.uploadFile.bind(this));
}
//----Get Lookup tables dynamically-----------//
async uploadFile(req, res) {
const blobServiceClient = await BlobServiceClient.fromConnectionString(this.AZURE_STORAGE_CONNECTION_STRING);
// Create a unique name for the container
//const containerName = 'quickstart' + uuidv1();
const containerName = blobServiceClient.getContainerClient('mycontainer');
console.log('\t', containerName.containerName);
// Get a reference to a container
const containerClient = await blobServiceClient.getContainerClient(containerName.containerName);
let form = new formidable.IncomingForm();
form.parse(req, async function (err, fields, files) {
const blobName = 'test' + uuidv1() + files.file;
// Get a block blob client
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
console.log('\nUploading to Azure storage as blob:\n\t', blobName);
// Upload data to the blob
const data = 'Hello test';
const uploadBlobResponse = await blockBlobClient.upload(data, data.length);
console.log("Blob was uploaded successfully. requestId: ", uploadBlobResponse.requestId);
});
}
}
module.exports = BlobController
could anyone help me on how can I upload files posted to Azure blob using Node.js
You were almost there :).
Please change your following code:
form.parse(req, async function (err, fields, files) {
const blobName = 'test' + uuidv1() + files.file;
// Get a block blob client
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
console.log('\nUploading to Azure storage as blob:\n\t', blobName);
// Upload data to the blob
const data = 'Hello test';
const uploadBlobResponse = await blockBlobClient.upload(data, data.length);
console.log("Blob was uploaded successfully. requestId: ", uploadBlobResponse.requestId);
});
to:
form.parse(req, async function (err, fields, files) {
const file = files.file;
const blobName = 'test' + uuidv1() + files.file;
const contentType = file.type;
const filePath = file.path;//This is where you get the file path.
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadBlobResponse = await blockBlobClient.uploadFile(filePath);
});
//Here's my form.parse code I used to upload pictures.
form.parse(req, async (err: any, fields: any, files: any) => {
const file = files.file;
const filePath = file.path;//This is where you get the file path. (this is the file itself)
const blobName: string = slugify(file.name);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadBlobResponse = await blockBlobClient.uploadFile(filePath)
console.log("Blob was uploaded successfully. requestId: ", uploadBlobResponse)
if (err) return reject(err)
//write to DB
//end write to DB
resolve(fields)
})
For anyone trying to make use of streams, this worked for me:
import formidable from 'formidable';
import { PassThrough } from 'stream';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method == 'POST') {
const stream = new PassThrough();
const form = new formidable.IncomingForm({
fileWriteStreamHandler: () => {
return stream;
}
});
form.parse(req, (err, fields, files) => {
if (files) {
if (files['<form-file-input-name>']) {
const file = files['<form-file-input-name>'] as formidable.File;
const mimeType = file.mimetype;
const extension = file.originalFilename ? file.originalFilename.substring(file.originalFilename.lastIndexOf('.')) : '.csv';
const newFileName = `<form-file-input-name>-${new Date().toISOString()}${extension}`;
getFilesBlobContainer().getBlockBlobClient(newFileName).uploadStream(stream, undefined, undefined, {
blobHTTPHeaders: {
blobContentType: mimeType,
},
});
}
}
});
return res.status(200).end();
}
}
export const config = {
api: {
bodyParser: false, //Disable NextJS body parsing so formidable can do that itself (fails silently otherwise)
},
};

How to chain writeFile() and OCR with NodeJS in Google Cloud Functions?

The scenario is as follows:
From an Amazon S3 bucket a file is fetched, then it is stored in a temporary folder and then Object Character Recognition is to be performed using the API.
Unfortunately, this doesn't work, I think it's due to the asynchronous/synchronous execution, but I've already tried several variants with callbacks/promises and didn't get any further.
If someone can give me a hint on how to construct this scenario I would be grateful!
The current error is:
TypeError: Cannot read property 'writeFile' of undefined at Response.<anonymous> (/srv/index.js:38:32) (it's the 'await fs.writeFile(dir,data);' line)
/**
* Responds to any HTTP request.
*
* #param {!express:Request} req HTTP request context.
* #param {!express:Response} res HTTP response context.
*/
const AWS = require('aws-sdk');
const fs = require('fs').promises;
const Vision = require('#google-cloud/vision');
var os = require('os');
exports.helloWorld = async (req,res) => {
var bucket, fileName, fileUrl;
req.on('data', chunk => {
body += chunk.toString();
data.push(chunk);
});
req.on('end', () => {
bucket = JSON.parse(data).value1;
fileName = JSON.parse(data).value2;
fileUrl = JSON.parse(data).value3;
var s3 = new AWS.S3();
s3.getObject({
Bucket: bucket,
Key: fileName
},
async function(error, data) {
if (error != null) {
console.log("Failed to retrieve an object: " + error);
} else {
console.log("Loaded " + data.ContentType + " bytes");
var tmpdir = os.tmpdir();
var dir = tmpdir+'/'+fileName;
try{
await fs.writeFile(dir,data);
const vision = new Vision.ImageAnnotatorClient();
let text;
await vision
.textDetection('/tmp/' + fileName)
.then(([detections]) => {
const annotation = detections.textAnnotations[0];
console.log(1);
text = annotation ? annotation.description : '';
console.log(`Extracted text from image (${text.length} chars)`);
console.log(1);
console.log(text);
resolve("Finished ocr successfully");
})
.catch(error =>{
console.log(error);
reject("Error with OCR");
})
}catch(error){
console.log(error);
}
}
},
);
let message = bucket + fileName + fileUrl;
res.status(200).send(message);
});
};
You're getting that error, because you're running on an older version of Node (< 10.0.0), where fs.promises is not available. That's why fs is undefined, and you're getting:
TypeError: Cannot read property 'writeFile' of undefined at Response.<anonymous> (/srv/index.js:38:32) (it's the 'await fs.writeFile(dir,data);' line)
Either use a newer version, or just promisify the code.
const { promisify } = require('util');
const fs = require('fs');
// const fs = require('fs').promises
const writeFile = promisify(fs.writeFile);
And now use writeFile instead of fs.writeFile in your code.
Aside from that, there are a few issues with your code.
req.on('data', chunk => {
body += chunk.toString();
data.push(chunk);
});
data is not defined anywhere, and it doesn't make sense to push data into an array and then running JSON.parse on that array, given the next few lines.
bucket = JSON.parse(data).value1;
fileName = JSON.parse(data).value2;
fileUrl = JSON.parse(data).value3;
Furthermore, JSON.parse should be called only once, instead of parsing the same string (which is an array in your code, and will yield an error) 3 times.
const values = JSON.parse(body); // should be body instead of data with the posted code
bucket = values.value1;
fileName = values.value2;
fileUrl = values.value3;
This can be improved greatly by just posting bucket, fileName & fileUrl in the JSON instead of valueN.
const { bucket, fileName, fileUrl } = JSON.parse(body);
The whole code can be rewritten into:
const AWS = require('aws-sdk');
const { promisify } = require('util');
const fs = require('fs');
const Vision = require('#google-cloud/vision');
const os = require('os');
const path = require('path');
const writeFile = promisify(fs.writeFile);
exports.helloWorld = async (req,res) => {
let body = '';
req.on('data', chunk => {
body += chunk.toString();
});
req.on('end', async() => {
// post { "bucket": "x", "fileName": "x", "fileUrl": "x" }
const { bucket, fileName, fileUrl } = JSON.parse(body);
var s3 = new AWS.S3();
try {
const data = await s3.getObject({
Bucket: bucket,
Key: fileName
}).promise();
const tmpdir = os.tmpdir();
const filePath = path.join(tmpdir, fileName)
await writeFile(filePath, data);
const vision = new Vision.ImageAnnotatorClient();
const [detections] = await vision.textDetection(filePath)
const annotation = detections.textAnnotations[0];
const text = annotation ? annotation.description : '';
console.log(`Extracted text from image (${text.length} chars)`);
let message = bucket + fileName + fileUrl;
res.status(200).send(message);
} catch(e) {
console.error(e);
res.status(500).send(e.message);
}
});
};
NOTE: I don't know if Vision API works like this, but I used the same logic and parameters that you're using.

Resources