Getting the widths of a lot of images causes ENOTFOUND in Node.js - node.js

I have several sources of lists of images (flicker, images stored at s3, imgur, etc)
I want to get the dimenions of these images.
I use node and https://github.com/nodeca/probe-image-size to go over each url and use that to get the width of the image and count how many images are at a certain width via the following code
probes = [];
_.forEach(image_urls, url => {
probes.push(probe(url));
});
results = await Promise.all(probes);
_.forEach(results, result_of_image => {
width = parseInt(result_of_image.width / 10) * 10;
if (!widthes[width]) {
widthes[width] = 1;
} else {
widthes[width]++;
}
});
even though all urls are accessible, I sometimes get getaddrinfo ENOTFOUND with the stack
at ClientRequest.req.once.err (/image_script/node_modules/got/index.js:73:21)
at Object.onceWrapper (events.js:293:19)
at emitOne (events.js:101:20)
at ClientRequest.emit (events.js:191:7)
at TLSSocket.socketErrorListener (_http_client.js:358:9)
at emitOne (events.js:96:13)
at TLSSocket.emit (events.js:191:7)
at connectErrorNT (net.js:1031:8)
at _combinedTickCallback (internal/process/next_tick.js:80:11)
at process._tickDomainCallback (internal/process/next_tick.js:128:9)
I suspect that because the url list is very large (in the thousands) that node just takes all resources of the system and things just stop working properly (this is a guess)
Is there a better way to do the above? or provide node with some connection pool?

Related

Inserting json into postgresql using a for loop doesn't insert all my data?

so I've been working on inserting my JSON data into google cloud postgresql for a few days now and am running across an issue where not even close to all of my data is inserted. Here is my code:
//prior to this i am connecting to cloud using pg and creating tables InterestClubs and FilterClubs
//alldata a json file, an array of about 3000 objects
let count = 0; //incrementing this every time i loop through
for(const club of alldata){
client.query("INSERT INTO InterestClubs (name, clublink, university, description, logo, interests) VALUES ('"+club.title+"', '"+club.clubLink+"', '"+club.university+"', '"+club.descriptionSnippet+"', '"+club.logoLink+"', '"+club.interests+"')")
client.query("INSERT INTO FilterClubs (name, clublink, university, description, logo, polfilters, relfilters, culfilters) VALUES ('"+club.title+"', '"+club.clubLink+"', '"+club.university+"', '"+club.descriptionSnippet+"', '"+club.logoLink+"', '"+club.politicalFilters+"', '"+club.religiousFilters+"', '"+club.culturalFilters+"')");
count++;
}
console.log(count); //outputs 3000 (or however many clubs there are in the json file)
I seem to be successfully looping through the data 3000 times (leading me to believe that I have inserted 3000 objects), but when I run a query such as SELECT * FROM InterestClubs (using either node/express or the cloud shell), I only receive 19 objects in return. I thought it may have something to do with having to wait a certain amount of time to allow the client.queries to successfully connect and insert, but when I used async/await (awaiting in front of each query), nothing changed. Also, I am getting this error every time I try and insert (after the count is printed)
3611
events.js:174
throw er; // Unhandled 'error' event
^
error: syntax error at or near "s"
at Connection.parseE (C:\Users\User\Documents\Code\Personal_Projects\clubhaus\node_modules\pg\lib\connection.js:539:11)
at Connection.parseMessage (C:\Users\User\Documents\Code\Personal_Projects\clubhaus\node_modules\pg\lib\connection.js:366:17)
at Socket.<anonymous> (C:\Users\User\Documents\Code\Personal_Projects\clubhaus\node_modules\pg\lib\connection.js:105:22)
at Socket.emit (events.js:198:13)
at addChunk (_stream_readable.js:288:12)
at readableAddChunk (_stream_readable.js:269:11)
at Socket.Readable.push (_stream_readable.js:224:10)
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
Emitted 'error' event at:
at Query.handleError (C:\Users\User\Documents\Code\Personal_Projects\clubhaus\node_modules\pg\lib\query.js:108:8)
at Connection.emit (events.js:198:13)
at Socket.<anonymous> (C:\Users\User\Documents\Code\Personal_Projects\clubhaus\node_modules\pg\lib\connection.js:109:12)
at Socket.emit (events.js:198:13)
[... lines matching original stack trace ...]
at TCP.onStreamRead [as onread] (internal/stream_base_commons.js:94:17)
This makes me think that something about the 19th object could be triggering a syntax error, causing the queries to stop inputting but still looping through them, but I'm not sure. Any help would be appreciated!!
Check if that 20th object has unescaped quote characters in one of the properties you are using. If u are using npm package pg you can automatically escape those characters passing variable arguments to your insert statements like this:
client.query("INSERT INTO InterestClubs (name, clublink, university, description, logo, interests) VALUES ($1, $2, $3, $4, $5, $6)", [club.title, club.clubLink, club.university, club.descriptionSnippet, club.logoLink, club.interests])

How to parse error parameter in request callback?

I'm deliberately triggering an error in a stored procedure under certain conditions which I want to catch in my Node.js API which uses the Tedious package.
Code Snippet from API:
let request = new Request(sql, (err)=>{
if (err) {
sqlerr = err;
console.log(typeof(err));
console.log("**RQ-ERROR**", err);
}
});
In the callback of the "Request" object above there is an "err" parameter. The "typeof()" returns "object"; however, when I dump it to the console it looks like this:
**RQ-ERROR** { RequestError: Duplicate entry for specified period
at RequestError (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\tedious\lib\errors.js:32:12)
at Parser.tokenStreamParser.on.token (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\tedious\lib\connection.js:723:34)
at emitOne (events.js:96:13)
at Parser.emit (events.js:188:7)
at Parser.parser.on.token (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\tedious\lib\token\token-stream-parser.js:27:14)
at emitOne (events.js:96:13)
at Parser.emit (events.js:188:7)
at addChunk (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\readable-stream\lib\_stream_readable.js:297:12)
at readableAddChunk (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\readable-stream\lib\_stream_readable.js:279:11)
at Parser.Readable.push (C:\inetpub\wwwroot\PersonnelApps\kudosapi\node_modules\readable-stream\lib\_stream_readable.js:240:10)
message: 'Duplicate entry for specified period',
code: 'EREQUEST',
number: 50000,
state: 1,
class: 16,
serverName: 'PERSODG2LNN52\\SQLEXPRESS',
procName: 'CreateStatusReport',
lineNumber: 44 }
This almost looks like a JavaScript object but, as you can see, the "RequestError" data isn't quoted nor is there a comma after the text "240:10)" just before the "message" member. I'm not sure if this is a bug in TDS or if I'm just missing something but I cannot access any of the members as it is. I'd have to convert it to a string and parse it which is fine but isn't very elegant.
Suggestions?
as you can see, the "RequestError" data isn't quoted nor is there a comma after the text "240:10)"
These are artifacts of the console logging out the error message. You can try it out for yourself with something like the following:
$ node
> console.log(new Error('this is an error object!'));
Error: this is an error object!
at repl:1:13
at Script.runInThisContext (vm.js:119:20)
at REPLServer.defaultEval (repl.js:332:29)
at bound (domain.js:395:14)
at REPLServer.runBound [as eval] (domain.js:408:12)
at REPLServer.onLine (repl.js:639:10)
at REPLServer.emit (events.js:194:15)
at REPLServer.EventEmitter.emit (domain.js:441:20)
at REPLServer.Interface._onLine (readline.js:290:10)
at REPLServer.Interface._line (readline.js:638:8)
I'm not exactly sure what the desired outcome of this question is, but try inspecting the err.message property rather than using the typeof operator.

MongoError: connection 4 to cluster closed

I have a function in NodeJS using Mongoose driver like below:
Pseudocode:
function someFn(someParams) {
// Step 1: a couple of very fast mongo queries (in milliseconds)
// Step 2: HUGE CPU processing - think millions of data grouped, mapped, etc. (takes about a minute)
// Step 3: another mongo query which inserts the results from Step 2 into a collection
}
At step 3, I get the following error:
MongoError: connection 4 to cluster closed
at Function.MongoError.create (/home/some-user/my-repo/node_modules/mongodb-core/lib/error.js:29:11)
at TLSSocket.<anonymous> (/home/some-user/my-repo/node_modules/mongodb-core/lib/connection/connection.js:202:22)
at Object.onceWrapper (events.js:293:19)
at emitOne (events.js:101:20)
at TLSSocket.emit (events.js:191:7)
at _handle.close (net.js:513:12)
at Socket.done (_tls_wrap.js:332:7)
at Object.onceWrapper (events.js:293:19)
at emitOne (events.js:101:20)
at Socket.emit (events.js:191:7)
at TCP._handle.close [as _onclose] (net.js:513:12)
My MongoDB connection params are as follows:
mongoose.connect(connStr, {
server: {
socketOptions: {
keepAlive: 300000,
connectTimeoutMS: 300000,
socketTimeoutMS: 300000,
auto_reconnect: true
}
}
});
I don't understand why I'm getting this error at Step 3. Can someone help me out with this, please?
Figured out the issue after hours of debugging. My Step 3 mongoose query had too many documents(in the order of millions, from Step 2). The error from mongoose gives no reason why the connection is closing. A message like Too many documents or Too large query would've gone a long way in saving a lot of time.

Neo4j's NPM component blows up without error

I'm using the thingdom/node-neo4j module like this:
var neo = require('neo4j');
var db = new new.GraphDatabase(...);
but when I call the cypher method with some invalid script, instead of returning an error (allowing me to handle it), it blows up:
db.cypher('// invalid cypher script', (err, res) => {
if (err) console.log('ERROR: ' + err);
console.log(res);
})
what I get is a stack trace that looks like this:
/Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/errors.js:20
Error.captureStackTrace(this, this.constructor);
^
TypeError: Error.captureStackTrace is not a function
at ClientError.Error [as constructor] (/Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/errors.js:20:13)
at new ClientError (/Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/errors.js:81:48)
at Function.__dirname.Error.Error._fromObject (/Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/errors.js:70:14)
at /Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/GraphDatabase.js:302:25
at Request._callback (/Users/ekkis/dev/test/inc/node_modules/neo4j/lib-new/GraphDatabase.js:92:20)
at Request.self.callback (/Users/ekkis/dev/test/inc/node_modules/request/request.js:187:22)
at emitTwo (events.js:106:13)
at Request.emit (events.js:191:7)
at Request. (/Users/ekkis/dev/test/inc/node_modules/request/request.js:969:12)
at emitNone (events.js:91:20)
at IncomingMessage.emit (events.js:185:7)
at endReadableNT (_stream_readable.js:974:12)
at _combinedTickCallback (internal/process/next_tick.js:74:11)
at process._tickCallback (internal/process/next_tick.js:98:9)
I'm using node v7.3.0 and posted an issue with the maintainer here: https://github.com/thingdom/node-neo4j/issues/214 but so far no solution. is anyone else running into this? any solutions?

Failed to launch renderer:Exception For HTML5 to PDF Convertor

I am using "html5-to-pdf" module to convert HTML5 page to PDF.Below is the code snippet I am using for specifying the Input and Output files.
var html5pdf = require("html5-to-pdf");
var fs = require("fs");
fs.createReadStream("/Vineet/POC/HTML_Inputs/TestSample.html")
.pipe(html5pdf())
.pipe(fs.createWriteStream("/Vineet/POC/PDF_Outputs/Output.pdf"));
While executing this code I am getting below exception:
Error: Failed to launch renderer
at C:\Vineet\POC\node_modules\html5-to-pdf\src\phantom.coffee:55:35
at Proto.apply (C:\Vineet\POC\node_modules\dnode-protocol\index.js:123:13)
at Proto.handle (C:\Vineet\POC\node_modules\dnode-protocol\index.js:99:19)
at D.dnode.handle (C:\Vineet\POC\node_modules\dnode\lib\dnode.js:140:21)
at D.dnode.write (C:\Vineet\POC\node_modules\dnode\lib\dnode.js:128:22)
at SockJSConnection.ondata (stream.js:31:26)
at emitOne (events.js:96:13)
at SockJSConnection.emit (events.js:188:7)
at Session.didMessage (C:\Vineet\POC\node_modules\sockjs\lib\transport.js:220:25)
at WebSocketReceiver.didMessage (C:\Vineet\POC\node_modules\sockjs\lib\trans-websocket.js:102:40)
at C:\Vineet\POC\node_modules\sockjs\lib\trans-websocket.js:75:22
at .<anonymous> (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket\api\event_target.js:41:7)
at Array.forEach (native)
at EventTarget.dispatchEvent (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket\api\event_target.js:40:33)
at API.receive (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket\api.js:30:10)
at instance._emitFrame (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket\hybi_parser.js:285:44)
at instance.parse (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket\hybi_parser.js:143:18)
at Socket.<anonymous> (C:\Vineet\POC\node_modules\faye-websocket\lib\faye\websocket.js:72:33)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at readableAddChunk (_stream_readable.js:172:18)
at Socket.Readable.push (_stream_readable.js:130:10)
at TCP.onread (net.js:542:20)
Due to which The PDF generated is of 0 bytes and is not being able to open.CAn any one help me in resolving this issue.
This package need PhatomJS installed on your computer. Phantomjs renders the page and saves it to a PDF. Read html5-to-pdf options params
You should use html-pdf package instead.
var pdf = require('html-pdf'),
fs = require('fs');
pdf.create(fs.readFileSync('./Vineet/POC/HTML_Inputs/TestSample.html', 'utf8'), { format: 'Letter' }).toFile('./Vineet/POC/PDF_Outputs/Output.pdf', function(err, res) {
if (err) return console.log(err);
console.log(res);
});

Resources