I am running into a sync/async issue with my program and am having a hard time implementing async into my middleware. I have tried a few times and broken it a bunch, what would be a fairly simple way to implement async await?
Here is the route
router.post("/testmedia", uploadFiles, testMedia);
Here is the middleware
const uploadFiles = (req,res,next)=> {
let b2fileIDs = [];
let savedPhotoLinks = {};
upload(req,res,function(err){
req.files.forEach(function(image){
var b2accountId;
var b2authToken;
var b2uploadURL;
//console.log(image);
b2.authorize()
.then(function(auth){
b2accountId = auth.accountId;
b2authToken = auth.authorizationToken;
}).then(function(ret){
b2.getUploadUrl(process.env.B2BUCKET_ID)
}).then(function(cb){
b2uploadURL = cb.uploadUrl;
b2authToken = cb.authorizationToken;
}).then(function(cb){
b2.uploadFile({
uploadUrl: b2uploadURL,
uploadAuthToken: b2authToken,
filename: 'test',
data: image.buffer
}).then(function(cb){
var uploadLink = {link: cb.fileId};
b2fileIDs.push(uploadLink);
}).then(function(){
successFn(response){
console.log(b2fileIDs);
//Save ID's to DB here
}
}).catch((error) => {
console.log("This is an ERROR " + error );
});
})
console.log("1");
console.log('2');
})
console.log('3');
console.log(b2fileIDs);
})
console.log('4');
next();
};
The problem is that when I upload multiple files the async nature moves on without capturing the b2fileIDs line.
Related
I'm fairly new to Node and Express and am struggling with creating a route that takes a user uploaded file and processes it into another file. The problem is that the second time the user sends a request, I am having to wait for the first process to complete to allow the user to upload a new file.
The basic structure of route that I have is below
THE PROBLEM is that the function convertFile below is a time taking process and it keeps the server busy from accepting new requests. How do I make it so that once the project is saved in mongo db at newProject.save() - the process to convertFile runs in the background while the server accepts new requests from the same route?
I'm sending a response back the user after newProject.save() and was hoping that would allow the user to send another request. And although this sends another request, the server doesn't accept it since its busy with the previous process to convertFile
router.post('/', upload.fields([{ name: 'zip' }]), async (req, res, next) => {
let data = {
title: req.body.title,
description: req.body.description,
}
if (req.files && req.files.zip) {
const newProject = new MongoProject(data);
newProject.save()
.then(async (project) => {
res.send(project);
console.log("Project created");
const uploadedFilePath = path.normalize(req.files.zip[0].path);
// below method - "convertFile" is a time taking method
const extractZipinfo = await convertFile(uploadedFilePath , data.masterFile).then((zipInfo) => {
console.log({ zipInfo })
data.zipInfo = {
sizes: zipInfo.sizes
}
})
})
.catch(err => {
console.log("Error creating project");
return res.send(err);
})
}
})
Below is the simplified version of code in convertFile function (code modified for brevity):
I know that this can be improvised a lot, but i'm struggling with getting it to function as expected first (allowing multiple routes)
async function convertFile(inputFilePath, outputInfo) {
const outputFilePath = "output.abc";
const jsonFilePath = "output.json";
const doc = new Document(); // this is a class to store all data of the output file that we will write at the end
const _FileAPI = new fileAPI();
const outputFinalData = await _FileAPI.Init() // this is an async method
.then(() => {
const dataClass = initiateClass(); // this is a class to store data in JSON format
const paragraphs = _FileAPI.GetallParagraphs(inputFilePath);
for (let i = 0, len = paragraphs.size(); i < len; i++) {
for (let j = 0, lenj = paragraphs.size(); j < lenj; j++) {
const para = paragraphs.get(j);
// read each para and Capitalize each word
dataClass.paragraphs.push(para);
}
}
fs.writeFileSync(jsonFilePath, JSON.stringify(dataClass, null, 2), 'utf-8');
console.log("then")
}).then(() => {
const io = new NodeIO(); // this class helps in writing the file in the desired output format
const outData = io.write(outputFilePath, doc).then(() => {
outputInfo.sizes.push(fs.statSync(outputFilePath).size);
return outputInfo;
});
return outData;
});
return outputFinalData;
}
I am trying to make a site for crypto data using coin-gecko's API.
They have 2 different end points for what i require and as such require 2 different URLs.
I had no problem using into the globalUrl to get data such as the total Market cap, volume, etc. which i was able to render into my ejs.
My problem is now i cannot use the other URL for this, seeing as I cannot make another get request, what would be the best way to get data from the topCoinsUrl such as say the "id" of bitcoin from the 2nd url please
const https = require('https');
const app = express();
app.get("/", function(req, res) {
const globalUrl = "https://api.coingecko.com/api/v3/global";
const topCoinsUrl = "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=100&page=1&sparkline=false&price_change_percentage=1h"
https.get(globalUrl , function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const globalMarket = JSON.parse(data);
const totalCryptocurrencies = globalMarket.data.active_cryptocurrencies
let totalMarketCap = globalMarket.data.total_market_cap.usd
let totalMarketCapUsd = totalMarketCap.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let totalVolume = globalMarket.data.total_volume.usd
let total24hVolume = totalVolume.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let markets = globalMarket.data.markets
let bitcoinMarketShare = Math.round(globalMarket.data.market_cap_percentage.btc);
res.render("home", {
totalCryptocurrencies: totalCryptocurrencies,
totalMarketCap: totalMarketCapUsd,
total24hVolume: total24hVolume,
markets: markets,
bitcoinMarketShare: bitcoinMarketShare
});
})
}).on("error", function(error) {
console.error(error)
});
});
// Ideally i would like to add this to get the ID of bitcoin, but I get an error when i try to use the 2 get requests:
https.get(topCoinsUrl, function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const topCoinsUrl = JSON.parse(data);
let bitcoinId = topCoinsUrl[0].symbol
res.render("home", {
bitcoinId: bitcoinId
})
})
// Error handler
}).on("error", function(error) {
console.error(error)
});
});
If you wish to make 2 simultaneous requests, you should use something like Promise.all() . Create two network requests and fire them at the same time using Promise.all & collect their result.
You can use Blurebird as well... http://bluebirdjs.com/docs/api/promise.all.html
I'm building out a firebase function that uses the html-pdf package (which uses PhantomJS). The function works fine on my local machine, but whenever I deploy the function on Firebase, I get the following error:
Error: html-pdf: PDF generation timeout. Phantom.js script did not exit.
I've changed the timeout parameter for pdf.create() and keep getting the same result. Any idea on what might be creating this issue that is unique to only when a deploy this to Firebase? Code is below.
const pdf = require('html-pdf');
const runtimeOpts = {
timeoutSeconds: 540, // in seconds
memory: '2GB'
}
exports.sendToKindle = functions.runWith(runtimeOpts).https.onRequest(async (req, res) => {
// REMOVED A BLOCK OF CODE FOR SIMPLICITY, BUT CAN PUT BACK IN IF NEEDED//
var options = {
format: 'Letter',
directory: "/tmp",
timeout: 540000, // in milliseconds
};
const blookFileName = createFileName(blookData.title) + '.pdf';
const tempFilePath = path.join(os.tmpdir(), `${blookFileName}`);
const htmlFilePath = path.join(os.tmpdir(), 'book.html');
const htmlFs = fs.openSync(htmlFilePath, 'w');
await fs.promises.appendFile(htmlFilePath, bookHTML);
const fd = fs.openSync(tempFilePath, 'w');
var html = fs.readFileSync(htmlFilePath, 'utf8');
let mailgunObject = null;
pdf.create(html, options).toFile(tempFilePath, async (err, res) => {
if (err) return console.error(err);
mailgunObject = await sendEmail(tempFilePath, kindleEmail);
return console.log(res);
});
fs.closeSync(fd);
fs.closeSync(htmlFs);
return cors(req, res, () => {
res.status(200).type('application/json').send({'response': 'Success'})
})
I was able to solve this issue by modifying the code by having the pdf.create().toFile() placed within the return of the cloud function.
const pdf = require('html-pdf');
const runtimeOpts = {
timeoutSeconds: 300, // in seconds
memory: '1GB'
}
exports.sendToKindle = functions.runWith(runtimeOpts).https.onRequest(async (req, res) => {
// REMOVED A BLOCK OF CODE FOR SIMPLICITY, BUT CAN PUT BACK IN IF NEEDED//
var options = {
format: 'Letter',
directory: "/tmp",
timeout: 540000, // in milliseconds
};
const blookFileName = createFileName(blookData.title) + '.pdf';
const tempFilePath = path.join(os.tmpdir(), `${blookFileName}`);
const htmlFilePath = path.join(os.tmpdir(), 'book.html');
const htmlFs = fs.openSync(htmlFilePath, 'w');
await fs.promises.appendFile(htmlFilePath, bookHTML);
const fd = fs.openSync(tempFilePath, 'w');
var html = fs.readFileSync(htmlFilePath, 'utf8');
return cors(req, res, () => {
pdf.create(html, options).toFile(tempFilePath, async (err, res) => {
if (err) return console.error(err);
let mailgunObject = await sendEmail(tempFilePath, kindleEmail);
fs.closeSync(fd);
fs.closeSync(htmlFs);
return console.log(res);
});
res.status(200).type('application/json').send({'response': 'Success'})
})
I got the same issue. Actualy I realized that when I called the function using html-pdf through Postman, or simply through a request by Google Chrome, the pdf used to generate within a 2 or 3 seconds, whereas it was more 2 or 3 minutes when calling it directly from my app.
So this is what I did : putting html-pdf in a separate function that I deployed, and then calling it :
https = require('https');
https.get(https://us-central1-your-project-name.cloudfunctions.net/your-function-using-html-pdf)
For clarity I have other cloud functions that all run intermittently (i.e from 'cold' in around 2-6 seconds, and all use the same boilerplate set up of importing an admin instance and exporting the function as a module)
I've seen other similar posts but this is really bugging me. I have a cloud function like so:
const admin = require('../AdminConfig');
const { reportError } = require('../ReportError');
module.exports = (event) => {
const uid = event.params.uid;
const snapshot = event.data;
if (snapshot._newData === null ) {
return null;
}
console.log('Create org begin running: ', Date.now());
const organisation = event.data.val();
const rootRef = admin.database().ref();
const ref = rootRef.child('/organisations').push();
const oid = ref.key;
const userData = {
level: 'owner',
name: organisation.name,
};
const orgShiftInfo = {
name: organisation.name,
startDay: organisation.startDay || 'Monday',
};
const updatedData = {};
updatedData[`/users/${uid}/currentOrg`] = oid;
updatedData[`/users/${uid}/organisations/${oid}`] = userData;
updatedData[`/organisations/${oid}`] = organisation;
updatedData[`/org_shift_info/${oid}`] = orgShiftInfo;
rootRef.update(updatedData, (err) => {
if (err) {
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ error: true })
.then(() => {
console.log(`error adding organisation for ${uid}: `, err);
return reportError(err, { uid });
});
}
console.log('Create org wrote succesfully: ', Date.now());
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ success: true });
});
}
I understand the 'cold start' thing but I think something is seriously wrong that it's taking 25 seconds. The logs don't return any error and are as so:
Is there some deeper way I can debug this to try and figure out why it's taking so long? It's unusable at the moment. Thanks a lot.
Solved:
Sorry,
I misunderstood the API a bit. I should have watched the promise video first!
I needed to put
return rootRef.update...
instead of
rootRef.update...
I'm making a call to the server using resource and when I go to the base URL of
/viewEvent
It works fine. I receive all the database entries. However, when I go to
/viewEvent/1234
where 1234 is the eventID
I get a undefined is not a function and this is a crash from within angular. Stack trace is
TypeError: undefined is not a function
at copy (http://localhost:8000/js/lib/angular/angular.js:593:21)
at http://localhost:8000/js/lib/angular/angular-resource.js:410:19
at wrappedCallback (http://localhost:8000/js/lib/angular/angular.js:6846:59)
at http://localhost:8000/js/lib/angular/angular.js:6883:26
at Object.Scope.$eval (http://localhost:8000/js/lib/angular/angular.js:8057:28)
at Object.Scope.$digest (http://localhost:8000/js/lib/angular/angular.js:7922:25)
at Object.Scope.$apply (http://localhost:8000/js/lib/angular/angular.js:8143:24)
at done (http://localhost:8000/js/lib/angular/angular.js:9170:20)
at completeRequest (http://localhost:8000/js/lib/angular/angular.js:9333:7)
at XMLHttpRequest.xhr.onreadystatechange (http://localhost:8000/js/lib/angular/angular.js:9303:11) angular.js:575
When I examine the server, the request was made correctly. I can see that it got 1234 and it pulls the correct entry from the mongo database.
This is the controller logic
.controller("viewEventsController", ["$scope", 'EventService', '$location', function($scope, EventService, $location){
var path = $location.path().split('/');
var pathSize = path.length;
$scope.events = [];
if(pathSize === 2){
console.log("No event ID");
$scope.events = EventService.query();
}
else{
console.log("Event ID specified");
EventService.get({"eventID": path[pathSize - 1]}, function(data){
//$scope.events.push(data);
console.log(data);
}, function(error){
console.log(error);
});
}
}]);
and the service logic
service.factory('EventService', function($resource){
return $resource('api/viewEvent/:eventID');
});
It never makes it back to the controller so I'm "confident" it's not that. (watch it be that)
Not sure if the best way, but I got it working by doing
In service:
service.factory('EventService', function($resource){
return $resource('api/viewEvent/:eventID',
{eventID:"#eventID"},
{
'getSingleEvent': {
url: "api/viewEvent/:eventID",
method: "GET",
isArray: true
}
}
);
controller
var path = $location.path().split('/');
var pathSize = path.length;
EventService.getSingleEvent({"eventID":path[pathSize - 1]}, function(result){
$scope.updateEvent();
});
Server
routes = require('./routes')
var router = express.Router();
router.get('/api/viewEvent/:eventID', routes.viewEvent);
and in the routes directory I have a js file with
var mongoose = require('mongoose');
var db = mongoose.createConnection('localhost', 'eventApp');
var eventSchema = require('../models/createEvent.js').eventSchema;
var event = db.model('events', eventSchema);
exports.viewEvent = function(req, res){
console.log(req.params.eventID);
if(req.params.eventID) {
event.find({"_id": req.params.eventID}, function (error, events) {
console.log(events);
res.send(events);
});
}
else{
event.find({}, function (error, events) {
console.log(events);
res.send(events);
})
}
};