var content = 'Hi, welcome to my webpage.';
var options = {host: 'www.website.com', path: '/folder/song.mp3', agent:false};
http.get(options, function(r) {
r.on('data', function() {
if (r.statusCode !== '404') {
content += 'Download';
}
});
});
fs.writeFile('index.html', content);
So normally, if I want to write a static html webpage from a node.js script, this works perfectly. For some reason, however, if I try to append to content from within http.get, it doesn't work. The whole point is to check if the file/page exists from an external website, and if it does then to display a link to it. The code that checks for the existing file works just fine, but I can't append anything to an external variable it seems. Any help would be much appreciated.
You need to use end event of http to indicate when you finish receiving data.
As node is asynchronouse the fs.writeFile instruction is run before all your data is received.
Here's how you can do it:
http.get(options, function(r) {
r.on('data', function() {
if (r.statusCode !== '404') {
content += 'Download';
}
});
r.on('end', function() {
fs.writeFile('index.html', content);
});
});
Related
I want to get the html of this page for parsing(click the link to understand what content i want to get).
750-bond list
Here's my code to request this page content
var https = require("https");
var fs = require("fs");
var options = {
hostname: "www.prizebond.net",
port: 443,
path: "/dlist.php?num=455",
method: "GET"
};
var response = "";
var req = https.request(options, function (res) {
res.setEncoding("UTF-8");
console.log(res.statusCode);
res.on("data", function (chunk) {
response += chunk;
});
res.on("end", function () {
fs.writeFile("750-bond.html", response, function (err) {
if (err) {
console.log(err.message);
}
console.log("File downloaded");
});
console.log("end");
});
});
req.end();
Now the problem is that in my 750-bont.html file, I am getting the weird the
result of "Checking your browser before accessing the prizebond.net" not the
original content. Here's the screenshot what I got when I open the 750-
bond.html file in browser.
What I am doing wrong? And how can I get the original content of this webpage?
You can't, unless you write something more sophisticated, but you probably shouldn't.
The purpose of Cloudflare-protection is to prevent what you are trying to realize unfortunately.
You could look into a possibility to access whatever you want to access by a public API or something that prizebond.net provides for example.
I am using the Twilio Node Helper Library to make a call and record it.
According to the API link, GET should return a WAV file, but in my case it just returns a json with the recording metadata.
This is what I'm writing:
twilioClient = require('twilio')(config.twilio.acct_sid, config.twilio.auth_token)
var request = twilioClient.recordings('RE01234567890123456789012345678901')
get(function (err, recording){ // <- this "recording" is JSON
It doesn't matter if I tack on a '.mp3' to the end of the SID, I always get a JSON.
Ideally I want to write something like this:
var file = fs.createWriteStream('/Users/yasemin/Desktop/rec.mp3');
twilioClient.recordings('RE01234567890123456789012345678901')
.get(function (err, recording) {
if(!err){ recording.pipe(file); }});
Thanks!
I came across this and had to develop my own code to handle this.
Here is the code I came up with below
con.on('getvmx', function(data){
comModel.find({_id: data.id}, function(err, results){
var https = require('https');
var options = {
host: 'api.twilio.com',
port: 443,
path: '/2010-04-01/Accounts/' + sid + '/Recordings/'+ results[0].sid + '.mp3',
method: 'GET',
auth: sid + ":" + auth,
agent: false
};
var req = https.request(options, function(res) {
res.setEncoding('binary');
var mp3data = '';
res.on('data', function (chunk) {
mp3data += chunk;
});
res.on('end', function(){
try{
var fileName = "/var/www/tcc/public/vm/" + results[0].sid + '.mp3';
fs.writeFile(fileName, mp3data, 'binary', function(err){
if(err){
return console.log(err);
}else{
console.log("File Saved");
con.emit('vmload', results);
}
});
}catch(err){
console.log(err.message);
}
});
});
req.end();
console.log(results);
//load all messages
//load line from reach message
});
});
TLDR: Node Helper Library doesn't have recoded file downloading capability at the moment.
This is the response from Twilio Support:
Looking at the documentation on our web portal, you are certainly
correct, downloading the .wav or .mp3 is possible via API call.
However, from what I can see looking at the Node example code here:
https://www.twilio.com/user/account/developer-tools/api-explorer/recording
And the documentation from the Twilio-Node developer here:
http://twilio.github.io/twilio-node/#recordings
It looks to me like the helper library doesn't actually support direct
downloading, just viewing the recording data. You can download the
application through an HTTP call, as shown in the original docs link
you noted on your Stackoverflow question. Let me know if you need help
with that.
In the mean time, I've reached out to the author of the library to see
if this is by design or a feature to be added to the library. It's
open source of course, so you could make a pull and add it yourself if
you like!
I'm trying to read a PDF from a URL and display it to a user's browser (via the passed in 'response' object). I've tried to use the code below and it works sometimes, but generally fails:
function writePdfToBrowser(url, response) {
http.get(url, function(res) {
logger.verbose('about to start download...');
var chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
});
res.on("end", function() {
logger.verbose('downloaded');
var buffer = new Buffer.concat(chunks);
//write downloaded pdf to the original response
response.write(buffer);
//response.send(buffer);
response.end();
});
}).on("error", function() {
logger.error("error!");
});
}
In the new page where I attempted to load the pdf it would just say "Failed to load pdf".
I'm new to Node, so not sure where the problem lies, any ideas? Anyone have any working code to do the same thing?
Thank you for any help!
Mark
Use piping:
function pipe(url, res) {
var request = http.get(url, function(response) {
res.writeHead(response.statusCode, response.headers)
response.pipe(res);
});
request.on('error', function(error){
res.statusCode = 500;
res.end(error.message);
});
}
... and please provide next time more information about what and how it fails, some logs, inspect response im browser before. And so on..
I spawned the following child: var spw = spawn('ping', ['-n','10', '127.0.0.1']) and I would like to receive the ping results on the client side (browser) one by one, not as a whole.
So far I tried this:
app.get('/path', function(req, res) {
...
spw.stdout.on('data', function (data) {
var str = data.toString();
res.write(str + "\n");
});
...
}
and that:
...
spw.stdout.pipe(res);
...
In both cases browser waits 10 of the pings to complete, and then prints the result as a whole. I would like to have them one by one, how to accomplish that?
(Client is just making a call to .../path and console.logs the result)
EDIT: Although I do believe that websockets are necessary to implement this, I just want to know whether there are any other ways. I saw several confusing SO answers, and blog posts (in this post, at step one OP streams the logs to the browser) which didn't help, therefore I decided to go for a bounty for some attention.
Here's a complete example using SSE (Server sent events). This works in Firefox and probably Chrome too:
var cp = require("child_process"),
express = require("express"),
app = express();
app.configure(function(){
app.use(express.static(__dirname));
});
app.get('/msg', function(req, res){
res.writeHead(200, { "Content-Type": "text/event-stream",
"Cache-control": "no-cache" });
var spw = cp.spawn('ping', ['-c', '100', '127.0.0.1']),
str = "";
spw.stdout.on('data', function (data) {
str += data.toString();
// just so we can see the server is doing something
console.log("data");
// Flush out line by line.
var lines = str.split("\n");
for(var i in lines) {
if(i == lines.length - 1) {
str = lines[i];
} else{
// Note: The double-newline is *required*
res.write('data: ' + lines[i] + "\n\n");
}
}
});
spw.on('close', function (code) {
res.end(str);
});
spw.stderr.on('data', function (data) {
res.end('stderr: ' + data);
});
});
app.listen(4000);
And the client HTML:
<!DOCTYPE Html>
<html>
<body>
<ul id="eventlist"> </ul>
<script>
var eventList = document.getElementById("eventlist");
var evtSource = new EventSource("http://localhost:4000/msg");
var newElement = document.createElement("li");
newElement.innerHTML = "Messages:";
eventList.appendChild(newElement);
evtSource.onmessage = function(e) {
console.log("received event");
console.log(e);
var newElement = document.createElement("li");
newElement.innerHTML = "message: " + e.data;
eventList.appendChild(newElement);
};
evtSource.onerror = function(e) {
console.log("EventSource failed.");
};
console.log(evtSource);
</script>
</body>
</html>
Run node index.js and point your browser at http://localhost:4000/client.html.
Note that I had to use the "-c" option rather than "-n" since I'm running OS X.
If you are using Google Chrome, changing the content-type to "text/event-stream" does what your looking for.
res.writeHead(200, { "Content-Type": "text/event-stream" });
See my gist for complete example: https://gist.github.com/sfarthin/9139500
This cannot be achieved with the standard HTTP request/response cycle. Basically what you are trying to do is make a "push" or "realtime" server. This can only be achieved with xhr-polling or websockets.
Code Example 1:
app.get('/path', function(req, res) {
...
spw.stdout.on('data', function (data) {
var str = data.toString();
res.write(str + "\n");
});
...
}
This code never sends an end signal and therefore will never respond. If you were to add a call to res.end() within that event handler, you will only get the first ping – which is the expected behavior because you are ending the response stream after the first chunk of data from stdout.
Code Sample 2:
spw.stdout.pipe(res);
Here stdout is flushing the packets to the browser, but the browser will not render the data chunks until all packets are received. Thus the reason why it waits 10 seconds and then renders the entirety of stdout. The major benefit to this method is not buffering the response in memory before sending — keeping your memory footprint lightweight.
I have a node service that fetches a pdf from an API and serves that pdf.
When I curl or directly open the API, I do see the correct pdf.
But when I serve it from my Node app, I get an empty pdf.
Here's the section of my code that does the pdf render.
} else if (options.type === 'pdf') {
res.writeHead(200, {'content-type' : 'application/pdf', 'content-disposition': 'attachment; filename=invoice.pdf'});
res.end(data.invoice);
I've console.log'ed data.invoice to know it's the right stuff.
typeof(data.invoice) gives string; but I've also tried res.end(new Buffer(data.invoice)); which didn't work either.
Here's the section of my code that fetches the data
var http_options = {
method : options.method
, host : Config.API.host
, path : options.path
, port : Config.API.port
, headers : options.headers
};
var req = http.request(http_options, function (response) {
var raw_response = "";
response.on('data', function (response_data) {
raw_response += response_data.toString();
});
response.on('end', function () {
if (response.statusCode !== 200) {
cb(raw_response);
} else {
cb(false, raw_response);
}
});
});
req.setTimeout(timeout, function () {
req.abort();
cb("API connection timed out");
});
req.on('error', function (error) {
cb("API error while requesting for " + options.path + '\n' + error + '\n' + "http options: " + JSON.stringify(http_options)
});
req.end();
It's quite likely that the toString() and concatenation when you're receiving the PDF are corrupting it. Try writing raw_response to a file (you can use writeFileSync() since this is just a one-time test) and doing a byte-for-byte comparison with the same PDF retrieved with curl.
Note that if the process of string conversion has corrupted it, trying to convert it back to a buffer before sending it won't help. You'll have to keep the whole thing as a buffer from start to finish.
Since you don't intend to modify or read this data in transit, I suggest just using the pipe function to pipe all the data coming in from response out to req. this question has a good sample, but here's an excerpt.
req.on('response', function (proxy_response) {
proxy_response.pipe(response);
response.writeHead(proxy_response.statusCode, proxy_response.headers);
});
Note that there's no reason to convert the chunks coming in from the response from Buffers to something else, just write them through as unmodified buffers, and stream them (this is what pipe will do for you) instead of accumulating them to get maximum efficiency (and node.js streaming hipster points).