Node appendFile not appending data chunk to file on filesystem - node.js

I have a program that is trying to get the values from the request using curl and store them in a file and serve the stored content back. The decision to store or append the contents in file are based on a query parameter appendFlag
Now when i run this program what i am getting in console is "true" and "appending" This suggests that it indeed reads the flag goes to the if part but somehow the appendFile function is not working.
var http = require('http');
var url = require('url');
var querystring = require('querystring');
var fs = require('fs');
http.createServer(function(request,response){
var str = request.url.split('?')[1];
var query = querystring.parse(str);
var writeStream = fs.createWriteStream(query['fileName']);
console.log("query - ");
console.log(query["appendFlag"]);
request.on('data',function(chunk){
if(query["appendFlag"]=="true"){
console.log("appending");
fs.appendFile(query['fileName'],chunk.toString(),function(err){
if(err) throw err;
});
}else{
var bufferGood = writeStream.write(chunk);
if(!bufferGood) request.pause();
}
});
request.on('end',function(){
response.writeHead(200);
response.write("\n Content with this url is - \n");
var readStream = fs.createReadStream(query['fileName'],{bufferSize:64*1024});
readStream.on('data',function(chunk){
response.write(chunk.toString());
});
readStream.on('end',function(){
response.write("\n");
response.end();
});
});
writeStream.on('drain',function(){
request.resume();
});
}).listen(8080);
Then after reading an answer from SO( How to create appending writeStream in Node.js ) i tried -
// Program to extract url from the request and writing in that particular file
var http = require('http');
var url = require('url');
var querystring = require('querystring');
var fs = require('fs');
http.createServer(function(request,response){
var str = request.url.split('?')[1];
var query = querystring.parse(str);
var writeStream = fs.createWriteStream(query['fileName']);
options = {
'flags': 'a' ,
'encoding' : null,
'mode' : 0666
}
var appendStream = fs.createWriteStream(query['fileName'],[options]);
console.log("query - ");
console.log(query["appendFlag"]);
request.on('data',function(chunk){
if(query["appendFlag"]=="true"){
console.log("appending");
var bufferGood = appendStream.write(chunk.toString());
if(!bufferGood) request.pause();
}else{
var bufferGood = writeStream.write(chunk.toString());
if(!bufferGood) request.pause();
}
});
request.on('end',function(){
response.writeHead(200);
response.write("\n Content with this url is - \n");
var readStream = fs.createReadStream(query['fileName'],{bufferSize:64*1024});
readStream.on('data',function(chunk){
response.write(chunk.toString());
});
readStream.on('end',function(){
response.write("\n");
response.end();
});
});
writeStream.on('drain',function(){
request.resume();
});
}).listen(8080);
That is changed the flag to the 'a' and it also did not append the data?

Your can use your first variant. But before appendFile() you've opened writeStream for the same query["filename"]. The stream is already opened.
var writeStream = fs.createWriteStream(query['fileName']);
options = {
'flags': 'a' ,
'encoding' : null,
'mode' : 0666
}
var appendStream = fs.createWriteStream(query['fileName'],[options]);
May be it's better to do something like:
var options = {
flags: query.appendFile ? 'w' : 'a'
...
Next: why [options]? You should remove the brackets.
Next: there is no guarantee you'll have filename param in querystring. Please handle this situation.

Related

Output looped PIWIK API call to CSV

I've got the following problem: I am looping through API calls using different dates to append the output to a CSV file. However, the output data in the CSV file only contains data from the first date.
When I log the results to the command prompt I do get multiple dates, meaning the problem occurs when writing the output to the CSV.
Moment.js is used for setting the start and end date to loop through and fast-csv to write the output of the API call to a CSV file.
// load and configure
const piwik = require ('piwik').setup ('placeholderurl', 'XXXXX');
// filesystem requirement
var fs = require('fs');
// fast-csv requirement
var csv = require("fast-csv");
// moment.js requirement
var moment = require('moment');
// variabelen voor het loopen door datums
var a = moment().format('2016-05-12');
var b = moment().format('2016-05-15');
var stream = fs.createWriteStream ('my.csv', {flags: 'a'})
// samenstellen API url
for (var m = moment(a); m.isBefore(b); m.add(1, 'days')) {
piwik.api (
{
method: 'Live.getLastVisitsDetails',
idSite: 3,
period: 'day',
format: 'csv',
date: moment(m).format('YYYY-MM-DD')
},
function (err, data) {
if (err) {
console.log (err);
return;
}
console.log(data)
csv
.writeToStream(fs.createWriteStream("my.csv"), data, {flags: 'a', headers: true});
}
);
}
API token and url removed for privacy reasons.
Solved. Got rid of the PIWIK API package and decided to use HTTP GET to retrieve the url manually.
The code:
// http requirement
var http = require('http');
var request = require('request');
// filesystem requirement
var fs = require('fs');
// moment.js requirement
var moment = require('moment');
// variabelen voor het loopen door datums
var a = moment().format('2016-05-12');
var b = moment().format('2016-05-15');
var m = moment(a);
//var stream = fs.createWriteStream ('my.csv', {flags: 'a'})
// samenstellen API url
for (var m = moment(a); m.isBefore(b); m.add(1, 'days')) {
request
.get("http://placeholder.com/?module=API&method=Live.getLastVisitsDetails&idSite=3&period=day&date=" + moment(m).format('YYYY-MM-DD') + "&format=csv&token_auth=placeholdertoken&filter_limit=-1")
.on('error', function(err) {
console.log(err)
})
.pipe(fs.createWriteStream('data-' + moment(m).format('YYYY-MM-DD') + '.csv'))
console.log(moment(m).format('YYYY-MM-DD') + " " + "saved")
}

Parse XML from site and get values from elements

I am writing a NodeJS script to get the XML from a website and get the values from a specific element.
However, when running it it says 'undefined'.
Code:
var http = require('http');
var xml2js = require('xml2js');
var parseString = require('xml2js').parseString;
var req = http.get("http://feeds.nos.nl/nosnieuwsvideo?format=xml", function(res) {
// save the data
var xml = '';
res.on('data', function(chunk) {
xml += chunk;
console.log(xml);
});
res.on('end', function() {
parseString(xml, function(err, result) {
console.log(result['title']);
});
});
});
req.on('error', function(err) {
// debug error
});
What's the problem?
Thanks
Instead of result['title'], try result.rss.channel[0].title
you're using the wrong path to access it.
alternatively, you can only parse what you need
const transform = require('camaro')
const result = transform(xml, { title: '//channel/title'})
console.log(result.title)
Thanks everyone for the answers.
I got it working by using
for (var i = 0; i < objects2.length; i++) {
var title = Object.values(objects2[i].title).toString();
title.replace("['", "").replace("']", "");
var description = Object.values(objects2[i].description).toString();
description.replace("['", "").replace("']", "");
var pubtime = Object.values(objects2[i].pubDate).toString();
pubtime.replace("['", "").replace("']", "");
var link = Object.values(objects2[i].link).toString();
link.replace("['", "").replace("']", "");
console.log("\n");
console.log(pubtime);
console.log(link);
console.log(title);
console.log(description);
}

On which format send file to save it on gridfs?

Hy every one,
Please , i 'm study on a project using nodeJS, and i would like to know , in which format my node client must send the file to the server ( is it in base64 format or else ?).
my client is :
//client.js
$('#file').on('change', function(e){
encode64(this);
});
function encode64(input) {
if (input.files){
chap.emit('test', { "test" : input.files[0] });
var FR= new FileReader();
FR.readAsDataURL(input.files[0]);
FR.onload = function(e) {
chap.emit('test', { "test" : e.target.result } );
}
}
}
My server side is :
socket.on('test', function(e){
var gs = new gridStore(db, e.test,"w");
gs.writeFile(new Buffer(e.test,"base64"), function(err,calb){
if (!err)
console.log('bien passe');
else
console.log('erreur');
});
});
But this doesn't work , i get this error :
TypeError: Bad argument
at Object.fs.fstat (fs.js:667:11)
Any one could help me ?
Normally this is how you store into gridFs . I have used it to store files. hope it works.
fs = require('fs'),
var gfs = require('gridfs-stream');
var form = new multiparty.Form();
form.parse(req, function (err, fields, files) {
var file = files.file[0];
var filename = file.originalFilename; //filename
var contentType = file.headers['content-type'];
console.log(files)
var tmpPath = file.path ;// temporary path
var writestream = gfs.createWriteStream({filename: fileName});
// open a stream to the temporary file created by Express...
fs.createReadStream(tmpPath)
// and pipe it to gfs
.pipe(writestream);
writestream.on('close', function (file) {
// do something with `file`
res.send(value);
});
})

image upload with "post" in nodejs with request module

How do we upload an image received form the mobile app to another server using the request module in nodejs?
I have tried using the multipart module to extract the file and send it in formData attribute with the request module (post method). This doesn't seem to work.
Please Use following code, it has been tested in express. you can modify it according to your requirement
var path = require('path');
var util = require('util');
if (req.files.profile_image !== undefined) {
var file = req.files.profile_image;
var tmp_path = file.path;
var fileName = file.name;
var milliseconds = new Date().getTime();
var file_ext = fileName.substr((Math.max(0, fileName.lastIndexOf(".")) || Infinity) + 1);
var newFileName = requestedUser + '_' + milliseconds + '.' + file_ext;
var pathToFile = require('path').dirname(require.main.filename);
var mainPath = path.dirname(pathToFile)
var target_path = path.join(mainPath, 'public/uploads/users', newFileName);
var readStream = fs.createReadStream(tmp_path)
var writeStream = fs.createWriteStream(target_path);
util.pump(readStream, writeStream, function(err) {
if (err) {
//handle error
} else {
//successfully uploaded
}
});
} else {
//file not recieved
}
Thanks

MongoDB Time conversion

I have a MongoDB collection that I am querying based on a time frame and address number. If the query is successful, then the server will return a CSV file with a number of attributes stored in each query entry. This part is working fine.
The problem is that one of the attributes in my CSV file is the timestamp. I'd like it to return the local date time (ie. "Time":"2014-02-09T06:00:02.000Z")... however the CSV file is returning the date object in milliseconds (ie. "Time":1392040717774). Is there an easy way to transform the query entry before it's written to the CSV file to a local date string? Here's my code snippet:
var JSONStream = require('JSONStream');
var httpserver = http.createServer(function(req, res) {
var pathname = url.parse(req.url).pathname;
if (pathname=="/DownloadUCData") {
var requestData = '';
req.on('data', function (data) {
requestData += data;
});
req.on('end', function () {
var json = JSON.parse(requestData);
var st = new Date(json.startDate);
var et = new Date(json.endDate);
st.setHours(st.getHours()-4); //compensate for GMT offset
et.setHours(et.getHours()-4);
st=st.getTime();
et=et.getTime();
var proj=JSON.parse('{"state":1, "temperature":1, "mode":1, "speed":1, "time":1}');
var cursor = userControlCollection.find({"addr": json.addr.toString(), "time": {$gte:st, $lte:et}}, proj);
var dbstream = cursor.stream();
var tempname = json.type+".csv";
var wstream = fs.createWriteStream(tempname);
wstream.on('error', function(e){console.error(e);});
dbstream.on("end", function() {
wstream.end();
console.log("write end");
res.writeHead(200, {"Content-Type": "application/json"});
res.write(JSON.stringify({fname:tempname}));
res.end();
return;
});
var jsonToStrings = JSONStream.stringify(false);
dbstream.pipe(jsonToStrings).pipe(wstream);
});
}
So, I figured out one way to solve this problem (although there may be others). Basically, I had to add a transformation into the piping system to convert the .getTime() data into a new Date() object. Here's the code snippet which seemed to resolve the issue:
var Transform = require('stream').Transform;
var parser = new Transform({objectMode: true});
parser._transform = function(data, encoding, done) {
if(data.time) data.time = new Date(data.time);
this.push(data);
done();
};
var jsonToStrings = JSONStream.stringify(false);
dbstream.pipe(parser).pipe(jsonToStrings).pipe(wstream);

Resources