I am using Google Trends API to get historical trends data. I am reading keywords from a CSV file and firing google trends query.
Here is the code
var googleTrends = require('google-trends-api');
var fs = require('fs')
fs.readFile('merged.csv', 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
data = data.toString().split("\n");
recur(0, data);
});
function recur(index, data){
if (index < data.length){
var keyword = data[index].split(",")[0];
console.log(keyword);
googleTrends.trendData(keyword)
.then(function(results){
console.log(results);
index = index+1;
recur(index, data);
})
.catch(function(err){
console.error('We have an error!', err);
});
}
}
I get this error many times in between
[ Error: Quota limit exceeded, try again later
at parseJSON (/Users/shubhamjindal/Development/domain-info/node_modules/google-trends-api/lib/resources/htmlParser.js:48:9)
at /Users/shubhamjindal/Development/domain-info/node_modules/google-trends-api/lib/utils/trendData.js:26:11
at tryCatcher (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/util.js:16:23)
at Promise._settlePromiseFromHandler (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/promise.js:504:31)
at Promise._settlePromise (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/promise.js:561:18)
at Promise._settlePromise0 (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/promise.js:606:10)
at Promise._settlePromises (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/promise.js:685:18)
at Async._drainQueue (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/async.js:138:16)
at Async._drainQueues (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/async.js:148:10)
at Immediate.Async.drainQueues (/Users/shubhamjindal/Development/domain-info/node_modules/bluebird/js/release/async.js:17:14)
at runCallback (timers.js:570:20)
at tryOnImmediate (timers.js:550:5)
at processImmediate [as _immediateCallback] (timers.js:529:5) ]
Can someone suggest me some workaround this error?
Related
I am trying to download multiple files as zip in nodejs using archiver. This is my code:
exports.downloadAllFiles = function(req,res){
var archive = archiver('zip', {
gzip: true,
zlib: { level: 9 } // Sets the compression level.
});
var output = fs.createWriteStream( "/home/files/Downloads/demo.zip");
archive.pipe(output);
demoDb.findOne({ caseguid: req.params.id }, function(err, data) {
if (err) {
res.json(HttpStatus.INTERNAL_SERVER_ERROR, {});
} else {
if(data){
data.Files.forEach(function(singleDoc){
archive.append(fs.createReadStream(singleDoc.filePath), { name: singleDoc.fileName })
})
}
}
})
archive.finalize();
};
This is the error stack:
{ Error: queue closed
at Archiver.append (/home/node_modules/archiver/lib/core.js:552:24)
at Promise.<anonymous> (/home/server/controllers/caseController.js:1722:25)
at Promise.<anonymous> (/home/node_modules/mpromise/lib/promise.js:177:8)
at emitOne (events.js:96:13)
at Promise.emit (events.js:188:7)
at Promise.emit (/home/node_modules/mpromise/lib/promise.js:84:38)
at Promise.fulfill (/home/node_modules/mpromise/lib/promise.js:97:20)
at /home/node_modules/mongoose/lib/query.js:1406:13
at model.Document.init (/home/node_modules/mongoose/lib/document.js:254:11)
at completeOne (/home/node_modules/mongoose/lib/query.js:1404:10)
at Immediate.cb (/home/node_modules/mongoose/lib/query.js:1158:11)
at Immediate.<anonymous> (/homenode_modules/mquery/lib/utils.js:137:16)
at runCallback (timers.js:672:20)
at tryOnImmediate (timers.js:645:5)
at processImmediate [as _immediateCallback] (timers.js:617:5) message: 'queue closed', code: 'QUEUECLOSED', data: undefined }
Probably the line archive.finalize() is being executed before your callback be completed. Try to move your archive.finalize to inside your callback.
I am new for node js. I am trying to insert the data to database, before that I did some validation for checking username already exist in database. Please see the coding which is used by me.
var express = require("express");
var router = express.Router();
var models = require("../models");
const { check, validationResult } = require('express-validator/check');
/* GET users listing. */
router.get('/', function(req, res, next) {
res.send('respond with a resource - merchant');
});
/* POST ADD USERS. */
router.post('/add-merchant', [
check('name').not().isEmpty(),
], (req, res, next) => {
try {
const errors = validationResult(req);
if (!errors.isEmpty()) {
return res.json({ errors: errors.array()});
}
console.log("error passed")
models.merchants.findOne({ where: {name: req.body.name}}).then((merchant) => {
if(merchant) {
throw new Error("Username already exists")
}
models.merchants.create(req.body).then((merchant) => {
res.json(merchant)
});
})
} catch (error) {
res.json({"status": "error", "message": error.message})
}
});
module.exports = router;
I am getting below error while running the above code
Unhandled rejection Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
at ServerResponse.setHeader (_http_outgoing.js:470:11)
at ServerResponse.header (/home/assistanz/Desktop/node/gb360/node_modules/express/lib/response.js:767:10)
at ServerResponse.send (/home/assistanz/Desktop/node/gb360/node_modules/express/lib/response.js:170:12)
at ServerResponse.json (/home/assistanz/Desktop/node/gb360/node_modules/express/lib/response.js:267:15)
at models.merchants.create.then (/home/assistanz/Desktop/node/gb360/routes/merchant.js:33:21)
at tryCatcher (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/util.js:16:23)
at Promise._settlePromiseFromHandler (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/promise.js:512:31)
at Promise._settlePromise (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/promise.js:569:18)
at Promise._settlePromise0 (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/promise.js:614:10)
at Promise._settlePromises (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/promise.js:694:18)
at _drainQueueStep (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/async.js:138:12)
at _drainQueue (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/async.js:131:9)
at Async._drainQueues (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/async.js:147:5)
at Immediate.Async.drainQueues [as _onImmediate] (/home/assistanz/Desktop/node/gb360/node_modules/bluebird/js/release/async.js:17:14)
at runCallback (timers.js:705:18)
at tryOnImmediate (timers.js:676:5)
at processImmediate (timers.js:658:5)
Please help anyone to solve this issue.
Thanks in advance.
this error occurs because you return the response and something is remain to load
you have to remove console.log("error passed") or you can put it before return statement..
and write return statement for every res.json statement
as
return res.json({})
HTTP uses a cycle that requires one response per request. When the client sends a request the server should send only one response back to client. So modify the code as below,
models.merchants.findOne({ where: { name: req.body.name } }).then((merchant) => {
if (merchant) {
throw new Error("Username already exists")
} else {
models.merchants.create(req.body).then((merchant) => {
res.json(merchant)
});
}
})
I'm using this api to login to web.spaggiari.eu but when I try to do so it appears this error: "Unhandled rejection Error: Invalid session token".
Also this error appears below:
** at request.then.res (C:\Users\Marian\node_modules\ClassevivaAPI\index.js:28:26)
at tryCatcher (C:\Users\Marian\node_modules\bluebird\js\release\util.js:16:23)
at Promise._settlePromiseFromHandler (C:\Users\Marian\node_modules\bluebird\js\release\promise.js:512:31)
at Promise._settlePromise (C:\Users\Marian\node_modules\bluebird\js\release\promise.js:569:18)
at Promise._settlePromise0 (C:\Users\Marian\node_modules\bluebird\js\release\promise.js:614:10)
at Promise._settlePromises (C:\Users\Marian\node_modules\bluebird\js\release\promise.js:693:18)
at Async._drainQueue (C:\Users\Marian\node_modules\bluebird\js\release\async.js:133:16)
at Async._drainQueues (C:\Users\Marian\node_modules\bluebird\js\release\async.js:143:10)
at Immediate.Async.drainQueues (C:\Users\Marian\node_modules\bluebird\js\release\async.js:17:14)
at runCallback (timers.js:789:20)
at tryOnImmediate (timers.js:751:5)
at processImmediate [as _immediateCallback] (timers.js:722:5)**
This is where the error occours:
request(req)
.then(res => {
var result = {};
var cookies = parseCookies(res.caseless.dict['set-cookie']);
if (cookies.PHPSESSID) {
result.sessionId = cookies.PHPSESSID;
} else throw new Error('Invalid session token');
if (res.body.data.auth.verified && res.body.data.auth.loggedIn) {
return new Session(result.sessionId)
} else throw new Error(res.body.error[0] || res.body.data.auth.errors[0] || 'Failed to authenticate')
})
Note: I have changed the "uri" that is the link where it will do the request in the index.js file because the previous one thrown a 503 error.
I'm trying to to limit the rate at which I'm firing GET requests to an outside API using promises but I'm having difficulty getting it working. In my scenario, I'm using 'request-promise' module and I need to send GET requests for 175 items from an API (one request for each item ID). The API has a rate limit of 40 requests per 10 seconds so my throttle needs to be 250ms per request. I am trying to send a request inside loop for each item ID, like:
files.forEach(function (file, i) {
console.log("The item ID is " + file.match(re)[1]);
client.send(new APIClient.requests.getItem(file.match(re)[1]))
.then((item) => {
...
})
.catch((error) => {
console.error(error);
// Use fallback
});
...
Here is a snippet of my API client return a request promise (rp) with a 250ms timeout and no callback:
const rp = require('request-promise');
const rp_errors = require('request-promise/errors');
...
send(request, callback) {
...
return rp(options)
.then(this._parseResponse)
.then((response)=> {
return new Promise( (resolve) => setTimeout(function(){
if (callback) { return callback(null, response); }
return resolve(response);
}, 250));
})
.catch(rp_errors.StatusCodeError,((error) => {
throw new errors.ResponseError(request, error.statusCode, error.message);
}
))
.catch(rp_errors.RequestError,((error) => {
if(error.cause.code === 'ETIMEDOUT' || error.cause.code === 'ESOCKETTIMEDOUT')
throw new errors.TimeoutError(request, error);
throw error;
}
))
.catch((error) => {
if (callback) {return callback(error)};
throw error;
});
}
The Async doesnt work and it returns this stack trace of "429 request limit exceeded"
{ ResponseError: 429 - {"status_code":25,"status_message":"Your request count (175) is over the allowed limit of 40."}
[0] at rp.then.then.catch (/mnt/c/Users/ridhwaan/Source/homehost/lib/api-client.js:52:19)
[0] at tryCatcher (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/util.js:16:23)
[0] at /mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/catch_filter.js:17:41
[0] at tryCatcher (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/util.js:16:23)
[0] at Promise._settlePromiseFromHandler (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/promise.js:512:31)
[0] at Promise._settlePromise (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/promise.js:569:18)
[0] at Promise._settlePromise0 (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/promise.js:614:10)
[0] at Promise._settlePromises (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/promise.js:689:18)
[0] at Async._drainQueue (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/async.js:133:16)
[0] at Async._drainQueues (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/async.js:143:10)
[0] at Immediate.Async.drainQueues [as _onImmediate] (/mnt/c/Users/ridhwaan/Source/homehost/node_modules/bluebird/js/release/async.js:17:14)
[0] at runCallback (timers.js:756:18)
[0] at tryOnImmediate (timers.js:717:5)
[0] at processImmediate [as _immediateCallback] (timers.js:697:5)
[0] name: 'ResponseError',
[0] request:
[0] Movie {
[0] method: 'GET',
[0] path: '/movie/24428',
[0] timeout: 10000,
[0] ensureHttps: false,
[0] external_id: '24428' },
[0] statusCode: 429 }
So the main issue is that array.forEach is a synchronous function and will not wait for client.send to finish.
A solution would be to use bluebird.mapSeries (http://bluebirdjs.com/docs/api/promise.mapseries.html) to map through an array and wait for each iteration to finish.
Also don't forget to return the send-promise, so the mapSeries-function will know when it has resolved, so it knows when to start the next iteration:
bluebird.mapSeries(files, function(file){
return send(...)
}
Last a suggestion would be to replace the whole .then(... setTimeout...) part with just .delay(250). Request-Promise already uses bluebird promises so you can make use of their convenience functions http://bluebirdjs.com/docs/api/delay.html. Delay will automatically resolve the value from the previous promise
return rp(options)
.then(this._parseResponse)
.delay(250)
.catch(...)
Using request promise module to do a post API call. The code is to get 'city' detail from the google maps api JSON output as below:
var rp = require('request-promise');
............
app.get('/check_promise', function(req,res) {
var option1 = {
url: 'https://maps.googleapis.com/maps/api/geocode/json?address=Florence',
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
json: true
}
rp(option1)
.then(function(response) {
console.log(response.data.results[0].formatted_address);
})
.catch(function(error){
console.log(error);
});
});
But the
console.log(response.data.results[0].formatted_address);
gives the below error:
TypeError: Cannot read property 'results' of undefined
at C:\Users\Laxmi\devapp\server.js:208:33
at tryCatcher (C:\Users\Laxmi\devapp\node_modules\bluebird\js\release\util.j
s:16:23)
at Promise._settlePromiseFromHandler (C:\Users\Laxmi\devapp\node_modules\blu
ebird\js\release\promise.js:512:31)
at Promise._settlePromise (C:\Users\Laxmi\devapp\node_modules\bluebird\js\re
lease\promise.js:569:18)
at Promise._settlePromise0 (C:\Users\Laxmi\devapp\node_modules\bluebird\js\r
elease\promise.js:614:10)
at Promise._settlePromises (C:\Users\Laxmi\devapp\node_modules\bluebird\js\r
elease\promise.js:693:18)
at Async._drainQueue (C:\Users\Laxmi\devapp\node_modules\bluebird\js\release
\async.js:133:16)
at Async._drainQueues (C:\Users\Laxmi\devapp\node_modules\bluebird\js\releas
e\async.js:143:10)
at Immediate.Async.drainQueues (C:\Users\Laxmi\devapp\node_modules\bluebird\
js\release\async.js:17:14)
at runCallback (timers.js:789:20)
at tryOnImmediate (timers.js:751:5)
at processImmediate [as _immediateCallback] (timers.js:722:5)
How to resolve the error to console log the city data?
console.log(response.results[0].formatted_address)