I'm learning NodeJS and I encounter a basic issue. I'm trying to read a file line by line, and for each line I read to send an HTTP request to / + <the line> e.g.:
wlist.txt contents
line
line2
failed attempt:
const request = require('request') // for http request later
const readline = require('readline')
const fs = require('fs')
function fileLoader() {
const readInterface = readline.createInterface({
input: fs.createReadStream('C:\\etc\\code\\req\\wlist.txt'),
});
readInterface.on('line', function(line) {
return "test";
});
}
var aba = fileLoader();
console.log(aba); // undefined
My logic inserting fileLoader as a function and not "as is" is that I later on have a switch case that uses the file load to different purposes such as XML request or JSON request.. lets say:
switch (myArgs[0]) {
case 'json':
let myJSON = {username: 'val'};
request({
url: "http://192.168.1.2:3000",
method: "POST",
json: true,
body: myJSON
}, function (error, response, body){
console.log(response.headers)
console.log(response.body)
});
break;
case 'xml': .....
I'm fully aware theres something I missing, probably regarding async / promises or anything, but to really educate, may someone please go easy on me and show me the way? I've tried everything and just can't get a grasp of whats the problem..
I believe you would like to do something like this:
https://gist.github.com/EB-BartVanVliet/533d55eb17c97f2a12ed25f479786f4a
Essentially what I do is:
Parse the file, look for empty lines and remove those
I declare a async start function so that I can use await inside the for loop
Log the output
You can do simple like this:
var sendRequest = function (input) {
// Do whatever you want here
}
var lineReader = require('readline').createInterface({
input: require('fs').createReadStream('path_to_your_file')
});
lineReader.on('line', function (line) {
console.log('Line from file:', line);
sendRequest(line);
});
readline is asynchronous so chances are that console.log is being called before fileLoader has finished. Try using readline-sync if you are happy to block whilst the file is read.
Otherwise you should re-write so the the on('line',...) method performs the action you want to take with the line as it is read. (I think this is what you want - "read a file line by line, and for each line I read to send an HTTP request"). E.g.
on('line', (input) => { /* perform send http stuff/call function to do it */ } );
Or, if you only want to act when the whole file is read, you'll have to re-structure so that the file-read is wrapped in a promise (or use async/await).
Related
I'm making a simple word combinatiion website.
and as a final step, I need all possible word in one string
so I write code like this
const fs=require('fs');
const axios=require('axios')
function test(want){
const res=axios.get("http://api.dictionaryapi.dev/api/v2/entries/en/"+want);
const datapromise=res.then((res)=>res.data);
return datapromise
}
fs.readFile('./input.txt','utf-8',function(error,data){
//console.log("console log")
var array=data.toString().split("\n");
fs.writeFile("./log.txt","",(err)=>{});
var res=""
for(i in array){
test(array[i]).then((data)=>(data)=>res+=data[0].word+"<br>").catch(/*(data)=>console.log(data.code)*/);
}
console.log(res);
})
But this code isn't work. console.log(res); is executed first and followed by for loop.
How can I fix it?
Without knowing much about Axios I can tell that axios.get and therefore the test function is going to be async. This means console.log here will always run first here as a result. Test ends up returning a promise that will resolve at a later time.
I'd do something like this (assuming you don't have async/await available):
var res= "";
var promises = [];
for(i in array) {
promises.push(
test(array[i]).then((data) => res+=data[0].word + "<br>")
);
}
Promise.all(promises).finally(() => {
console.log(res);
});
Other notes:
The catch here is being called but nothing is being passed in - this may result in an error
The then has a nested function that I imagine wouldn't ever be called (data) => (data) => this is basically creating a 2nd nested function. I don't think it'd get called.
Classic embarrassing newbie question. Why doesn't my store variable contain any results? I know it is accumulating results along the way. I also know enough about nodejs to know it has to do with promises, single-threadedness, etc.
var readline = require('readline');
var fs = require("fs");
var pathToFile = '/scratch/myData.csv';
var rd = readline.createInterface({
input: fs.createReadStream(pathToFile),
// output: process.stdout,
console: false
});
var store = {};
rd.on('line', function(line) {
store[line] = 1;
// console.log (`store is now: ${JSON.stringify (store)}`);
});
console.log (`store is now: ${JSON.stringify (store)}`);
This has nothing to do with Promises, (Although, you can promisify it, if you like).
As you said, it is accumulating the results line by line, but this is hapening inside the scope of the callback function.
And if you want to make use of the data, you will have to call another function inside the callback function when the last line is called, (or maybe listen to a different event).
Something like the following:
var store = {};
rd.on('line', function(line) {
store[line] = 1;
// console.log (`store is now: ${JSON.stringify (store)}`);
})
.on('close', function() {
myFunc(store);
});
function myFunc(store){
console.log (`store is now: ${JSON.stringify (store)}`);
}
I need to allow the user of my app to download a file with Meteor. Currently what I do is when the user requests to download a file I enter into a "fileRequests" collection in Mongo a document with the file location and a timestamp of the request and return the ID of the newly created request. When the client gets the new ID it imediately goes to mydomain.com/uploads/:id. I then use something like this to intercept the request before Meteor does:
var connect = Npm.require("connect");
var Fiber = Npm.require("fibers");
var path = Npm.require('path');
var fs = Npm.require("fs");
var mime = Npm.require("mime");
__meteor_bootstrap__.app
.use(connect.query())
.use(connect.bodyParser()) //I add this for file-uploading
.use(function (req, res, next) {
Fiber(function() {
if(req.method == "GET") {
// get the id here, and stream the file using fs.createReadStream();
}
next();
}).run();
});
I check to make sure the file request was made less than 5 seconds ago, and I immediately delete the request document after I've queried it.
This works, and is secure(enough) I think. No one can make a request without being logged in and 5 seconds is a pretty small window for someone to be able to highjack the created request URL but I just don't feel right with my solution. It feels dirty!
So I attempted to use Meteor-Router to accomplish the same thing. That way I can check if they're logged in correctly without doing the 5 second open to the world trickery.
So here's the code I wrote for that:
Meteor.Router.add('/uploads/:id', function(id) {
var path = Npm.require('path');
var fs = Npm.require("fs");
var mime = Npm.require("mime");
var res = this.response;
var file = FileSystem.findOne({ _id: id });
if(typeof file !== "undefined") {
var filename = path.basename(file.filePath);
var filePath = '/var/MeteorDMS/uploads/' + filename;
var stat = fs.statSync(filePath);
res.setHeader('Content-Disposition', 'attachment; filename=' + filename);
res.setHeader('Content-Type', mime.lookup(filePath));
res.setHeader('Content-Length', stat.size);
var filestream = fs.createReadStream(filePath);
filestream.pipe(res);
return;
}
});
This looks great, fits right in with the rest of the code and is easy to read, no hacking involved, BUT! It doesn't work! The browser spins and spins and never quite knows what to do. I have ZERO error messages coming up. I can keep using the app on other tabs. I don't know what it's doing, it never stops "loading". If I restart the server, I get a 0 byte file with all the correct headers, but I don't get the data.
Any help is greatly appreciated!!
EDIT:
After digging around a bit more, I noticed that trying to turn the response object into a JSON object results in a circular structure error.
Now the interesting thing about this is that when I listen to the filestream for the "data" event, and attempt to stringify the response object I don't get that error. But if I attempt to do the same thing in my first solution(listen to "data" and stringify the response) I get the error again.
So using the Meteor-Router solution something is happening to the response object. I also noticed that on the "data" event response.finished is flagged as true.
filestream.on('data', function(data) {
fs.writeFile('/var/MeteorDMS/afterData', JSON.stringify(res));
});
The Meteor router installs a middleware to do the routing. All Connect middleware either MUST call next() (exactly once) to indicate that the response is not yet settled or MUST settle the response by calling res.end() or by piping to the response. It is not allowed to do both.
I studied the source code of the middleware (see below). We see that we can return false to tell the middleware to call next(). This means we declare that this route did not settle the response and we would like to let other middleware do their work.
Or we can return a template name, a text, an array [status, text] or an array [status, headers, text], and the middleware will settle the response on our behalf by calling res.end() using the data we returned.
However, by piping to the response, we already settled the response. The Meteor router should not call next() nor res.end().
We solved the problem by forking the Meteor router and making a small change. We replaced the else in line 87 (after if (output === false)) by:
else if (typeof(output)!="undefined") {
See the commit with sha 8d8fc23d9c in my fork.
This way return; in the route method will tell the router to do nothing. Of course you already settled the response by piping to it.
Source code of the middleware as in the commit with sha f910a090ae:
// hook up the serving
__meteor_bootstrap__.app
.use(connect.query()) // <- XXX: we can probably assume accounts did this
.use(this._config.requestParser(this._config.bodyParser))
.use(function(req, res, next) {
// need to wrap in a fiber in case they do something async
// (e.g. in the database)
if(typeof(Fiber)=="undefined") Fiber = Npm.require('fibers');
Fiber(function() {
var output = Meteor.Router.match(req, res);
if (output === false) {
return next();
} else {
// parse out the various type of response we can have
// array can be
// [content], [status, content], [status, headers, content]
if (_.isArray(output)) {
// copy the array so we aren't actually modifying it!
output = output.slice(0);
if (output.length === 3) {
var headers = output.splice(1, 1)[0];
_.each(headers, function(value, key) {
res.setHeader(key, value);
});
}
if (output.length === 2) {
res.statusCode = output.shift();
}
output = output[0];
}
if (_.isNumber(output)) {
res.statusCode = output;
output = '';
}
return res.end(output);
}
}).run();
});
I'm trying to do HTTP POST using the the request module from a node server to another server.
My code looks something like,
var req = request.post({url: "http://foo.com/bar", headers: myHeaders});
...
...
req.write("Hello");
...
...
req.end("World");
I expect the body of the request to be "Hello World" on the receiving end, but what I end up with is just "".
What am I missing here?
Note: The ellipsis in the code indicates that the write and the end might be executed in different process ticks.
It looks to me as if you are getting missed Request http.clientRequest/http.serverRequest
If you want to make a post to a server with request what you want to do is something like
request({ method:"post", url: "server.com", body:"Hello World"}, callback);
As 3on pointed, the correct syntax for a POST request is
request({ method:"post", url: "server.com", body:"Hello World"}, callback);
You also have a convenience method:
request.post({ url: "server.com", body:"Hello World"}, callback);
But from your question it seems like you want to stream:
var request = require('request');
var fs = require('fs');
var stream = fs.createWriteStream('file');
stream.write('Hello');
stream.write('World');
fs.createReadStream('file').pipe(request.post('http://server.com'));
Update:
You may break the chunks you write to the stream in any way you like, as long as you have the RAM (4mb is peanuts but keep in mind that v8 (the javascript engine behind node) has an allocation limit of 1.4GB I think);
You may see how much you "wrote" to the pipe with stream.bytesWritten where var stream = fs.createWriteStream('file') as you see in the piece of code above. I think you can't however know how much the other end of the pipe got, but bitesWritten should give you a pretty decent approximation.
You can listen to the data and end events of both stream and request.post('http://server.com')
I managed to make the code written in the question here valid and work as expected by modifying the request module a bit.
I noticed a block of code in request's main.js in the Request.prototype.init function (at line 356),
process.nextTick(function () {
if (self._aborted) return
if (self.body) {
if (Array.isArray(self.body)) {
self.body.forEach(function (part) {
self.write(part)
})
} else {
self.write(self.body)
}
self.end()
} else if (self.requestBodyStream) {
console.warn("options.requestBodyStream is deprecated, please pass the request object to stream.pipe.")
self.requestBodyStream.pipe(self)
} else if (!self.src) {
if (self.method !== 'GET' && typeof self.method !== 'undefined') {
self.headers['content-length'] = 0;
}
self.end();
}
self.ntick = true
})
I'm now overriding this function call by adding a new option (endOnTick) while creating the request. My changes: Comparing mikeal/master with GotEmB/master.
Is there a way that I can catch eventual console output caused by console.log(...) within node.js to prevent cloggering the terminal whilst unit testing a module?
Thanks
A better way could be to directly hook up the output you to need to catch data of, because with Linus method if some module write directly to stdout with process.stdout.write('foo') for example, it wont be caught.
var logs = [],
hook_stream = function(_stream, fn) {
// Reference default write method
var old_write = _stream.write;
// _stream now write with our shiny function
_stream.write = fn;
return function() {
// reset to the default write method
_stream.write = old_write;
};
},
// hook up standard output
unhook_stdout = hook_stream(process.stdout, function(string, encoding, fd) {
logs.push(string);
});
// goes to our custom write method
console.log('foo');
console.log('bar');
unhook_stdout();
console.log('Not hooked anymore.');
// Now do what you want with logs stored by the hook
logs.forEach(function(_log) {
console.log('logged: ' + _log);
});
EDIT
console.log() ends its output with a newline, you may want to strip it so you'd better write:
_stream.write = function(string, encoding, fd) {
var new_str = string.replace(/\n$/, '');
fn(new_str, encoding, fd);
};
EDIT
Improved, generic way to do this on any method of any object with async support See the gist.
module.js:
module.exports = function() {
console.log("foo");
}
program:
console.log = function() {};
mod = require("./module");
mod();
// Look ma no output!
Edit: Obviously you can collect the log messages for later if you wish:
var log = [];
console.log = function() {
log.push([].slice.call(arguments));
};
capture-console solves this problem nicely.
var capcon = require('capture-console');
var stderr = capcon.captureStderr(function scope() {
// whatever is done in here has stderr captured,
// the return value is a string containing stderr
});
var stdout = capcon.captureStdout(function scope() {
// whatever is done in here has stdout captured,
// the return value is a string containing stdout
});
and later
Intercepting
You should be aware that all capture functions will still pass the values through to the main stdio write() functions, so logging will still go to your standard IO devices.
If this is not desirable, you can use the intercept functions. These functions are literally s/capture/intercept when compared to those shown above, and the only difference is that calls aren't forwarded through to the base implementation.
Simply add the following snippet to your code will let you catch the logs and still print it in the console:
var log = [];
console.log = function(d) {
log.push(d);
process.stdout.write(d + '\n');
};