I have a working code - when it comes to uploading the file ::
var express=require('express');
var mysql=require('mysql');
var fs=require('fs');
var http=require('http');
var crypto=require('crypto');
var app=express();
var connection=mysql.createConnection({
host:'localhost',
user:'****',
password:'****',
database:'posting_only_image_DB'
});
connection.connect();
app.set('port',process.env.PORT||1234);
app.use(express.static(__dirname+'/public/images'));
app.use(express.bodyParser());
app.post('/Details/',function(req,res,next){
var file_name=req.files.key.originalFilename;
console.log(file_name);
crypto.randomBytes(8, function(ex, buf) {
var array = req.files.key.originalFilename.split('.');
var type = array[array.length - 1];
var name = buf.toString('hex') + '.' + type;
fs.rename(req.files.key.path, './public/images/' + name, function(e) {
next(e, name);
});
});
});
http.createServer(app).listen(app.get('port'),function(){
console.log('Express server listening on port'+app.get('port'));
});
I make a request as below while uploading the file::
however my query works and i can see the file with unique id in server disk in location /public/images .... with the file name:: 8ce456dc55575888.jpg
Now say if i want to test it and i want to view the uploaded file :: I five a Url request to below for the browser
http://-----URL----/Details/8ce456dc55575888.jpg
I get the error on browser as ::
Cannot GET /Details/8ce456dc55575888.jpg
How to resolve this ?
You don't have a GET route for /Details/8ce456dc55575888.jpg.
If you want the static middleware to handle those requests, you need to configure it properly:
app.use('/Details', express.static(__dirname+'/public/images'));
This means that requests for:
/Details/8ce456dc55575888.jpg
Will be mapped to:
{{__dirname}}/public/images/8ce456dc55575888.jpg
You can use that in addition to the other express.static that you're using.
Related
I am using post method in nodejs
Well it is working fine when I run it using postman
but when I run it in my browsers it shows error
Cannot GET /listUsers
and
listUsers:1 GET http://localhost:8081/listUsers 404 (Not Found)
this
here is my node js code
var express = require('express');
var bodyParser = require('body-parser');
var app = express();
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended:false}));
app.post('/listUsers', function (req, res) {
var f = parseInt(req.body.f);
console.log("hello" + f);
var l = parseInt(req.body.l);
var sum = Number(f + l);
res.send('The sum is: ' + Number(sum));
})
var server = app.listen(8081, function () {
var host = server.address().address
var port = server.address().port
console.log("Example app listening at http://%s:%s", host, port)
})
here is my jquery code from where I am sending data
usid();
function usid(med){
var f = "my new name";
$.ajax({
crossDomain: true,
url:"http://localhost:8081/listUsers",
method:"POST",
data:{med,f:f},
success:function(data,status){
console.log("send");
}
})
}
can I get where I am doing mistake?
You are not submitting any form/storing any resource. What you are trying should ideally be done with GET.
Reasons it is not working with browser:
Visiting a page by changing url is a GET method. You do not have a GET method defined in your app for this route. Only one POST method.
What you can do:
Change POST to GET both in server and your AJAX. Change both methods accordingly. You will then have to pass query params(google something and everything after the question mark is how query Params work).
I'm currently working on a Phonegap app, and I would like users to be able to upload any file to my NodeJS server.
I've looking all around the web but I just can't get anything to work...
Here is the code I'm using for the Phonegap controller:
$scope.open = function()
{
navigator.camera.getPicture(upload,
function(message)
{
alert('get picture failed');
},
{
quality: 50,
destinationType: navigator.camera.PictureSourceType.FILE_URI,
sourceType: navigator.camera.PictureSourceType.PHOTOLIBRARY,
mediaType: navigator.camera.MediaType.ALLMEDIA
});
}
var win = function (r) {
$scope.log = "Code = " + r.responseCode;
$scope.log2 = "Response = " + r.response;
$scope.log3 = "Sent = " + r.bytesSent;
$scope.$digest();
}
var fail = function (error) {
$scope.log = "An error has occurred: Code = " + error.code;
$scope.log2 = "upload error source " + error.source;
$scope.log3 = "upload error target " + error.target;
$scope.$digest();
}
function upload(fileURI)
{
$scope.log = fileURI;
$scope.$digest();
var options = new FileUploadOptions();
options.fileKey = "file";
options.fileName = fileURI.substr(fileURI.lastIndexOf('/') + 1);
options.mimeType = "text/plain";
options.chunkedMode = false;
var params = {};
params.value1 = "test";
params.value2 = "param";
options.params = params;
var ft = new FileTransfer();
ft.upload(fileURI, "http://192.168.192.111:2999/upload", win, fail, options);
}.
Here is the current code for the NodeJS server, have tried a lot of different things, all without success:
var express = require('express');
var http = require('http').Server(express);
var io = require('socket.io')(http);
var fs = require('fs');
var multer = require('multer');
var app = new express();
app.post('/upload', multer({dest: './uploads/'}).single('upl'), function(req, res)
{
console.log(req.body);
console.log(req.file);
})
http.listen(2999, function(){
console.log('listening on *:2999');
});
In the app I used to get errors that FileUploadOptions etc weren't defined, but I fixed that by adding them to the cordova project.
Furthermore, I use ionic 1, if that helps anyone out.
I do keep constantly getting the error code 1 (upload error source), even though I selected a real file and I saw that the link was correct (something like /storage/0/emulated/Downloads on my Android device).
Also, sometimes it gives me error 3 as well (upload target source), some sort of server not found issue I think.
Is there something obvious I'm doing wrong and how would I be able to fix it? Is there a handier way, since I eventually want to link this to a MySQL database.
Thanks in advance!
Well found my answer (a while ago, this is for people stumbling across this post).
You can first try whether your JS works by changing the server to https://posttestserver.com/post.php. If you see an upload appearing there, there's a problem with the server.
The problem with me was that I didn't let Apache through the firewall at all, so uploads from anything besides my PC would fail...
var express=require('express');
var bodyParser=require('body-parser');
var formidable = require('formidable');
var util = require('util');
var app=express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
var path=require('path');
var mysql =require('mysql');
var fs=require('fs');
app.use('/public',express.static(path.join(__dirname, 'public')));
var connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password : '',
port : 3306, //port mysql
database:'xxxxx'
});
app.post('/data', function(req, res) {
// create an incoming form object
var form = new formidable.IncomingForm(),
files = [],
fields = [];
// specify that we want to allow the user to upload multiple files in a single request
form.multiples = true;
// store all uploads in the /uploads directory
form.uploadDir = path.join(__dirname, '/public/images/uploads');
// every time a file has been uploaded successfully,
// rename it to it's orignal name
form.on('file', function(field, file) {
if (path.extname(file.name)=='') {
extension='.jpg';
}
else{
extension=path.extname(file.name);
}
var oldpath = file.path;
form.uploadDir = path.basename(file.name,extension).concat((Math.random()* 100),extension);
var newpath = './public/images/uploads/'+ path.basename(file.name,extension).concat((Math.random()* 100),extension);
//fs.rename(file.path, path.join(form.uploadDir, file.name));
fs.rename(oldpath, newpath);
});
form.on('field', function(field, value) {
fields[field] = value;
});
// log any errors that occur
form.on('error', function(err) {
console.log('An error has occured: \n' + err);
});
// once all the files have been uploaded, send a response to the client
//Call back at the end of the form.
form.on('end', function () {
res.writeHead(200, {
'content-type': 'text/plain'
});
res.write('received the data:\n\n');
// console.log(fields.name+'-'+fields.nickname);
var values={
name:fields.name,
nickname:fields.nickname,
email:fields.email,
password:fields.password,
dob:fields.dob,
gender:fields.gender,
phoneno:fields.phone
};
connection.query('INSERT INTO users SET ?', values, function(err,req,res){
if(err){
console.log('Connection result error '+err);
}
else{
console.log('Success');
}
});
res.end();
});
// parse the incoming request containing the form data
form.parse(req);
});
//app.use(app.router);
app.listen(5000);
I'm trying to parse and insert a big csv file into MongoDB but when the file extends 100'000 rows I get a bad response from the server. And the files I need to insert are usually above 200'000 rows.
I've tried both bulk insert (insertMany) and Babyparse(Papaparse) streaming approach to insert the file row by row. But with poor results.
Node api:
router.post('/csv-upload/:id', multipartMiddleware, function(req, res) {
// Post vartiables
var fileId = req.params.id;
var csv = req.files.files.path;
// create a queue object with concurrency 5
var q = async.queue(function(row, callback) {
var entry = new Entry(row);
entry.save();
callback();
}, 5);
baby.parseFiles(csv, {
header: true, // Includes header in JSON
skipEmptyLines: true,
fastMode: true,
step: function(results, parser) {
results.data[0].id = fileId;
q.push(results.data[0], function (err) {
if (err) {throw err};
});
},
complete: function(results, file) {
console.log("Parsing complete:", results, file);
q.drain = function() {
console.log('All items have been processed');
res.send("Completed!");
};
}
});
});
This streaming approach results in: POST SERVER net::ERR_EMPTY_RESPONSE
Not sure if I'm using the async.queue correctly though.
Is there a better and more efficient way to do this OR am I doing something wrong?
Express Server:
// Dependencies
var express = require('express');
var path = require('path');
var bodyParser = require('body-parser');
var routes = require('./server/routes');
var mongoose = require("mongoose");
var babel = require("babel-core/register");
var compression = require('compression');
var PORT = process.env.PORT || 3000;
// Include the cluster module
var cluster = require('cluster');
mongoose.connect(process.env.MONGOLAB_URI || 'mongodb://localhost/routes');
// Code to run if we're in the master process
if (cluster.isMaster) {
// Count the machine's CPUs
var cpuCount = require('os').cpus().length;
// Create a worker for each CPU
for (var i = 0; i < cpuCount; i += 1) {
cluster.fork();
}
// Code to run if we're in a worker process
} else {
// Express
var app = express();
app.use(bodyParser.json({limit: '50mb'}));
app.use(bodyParser.urlencoded({limit: '50mb', extended: true}));
// Compress responses
app.use(compression());
// Used for production build
app.use(express.static(path.join(__dirname, 'public')));
routes(app);
// Routes
app.use('/api', require('./server/routes/api'));
app.all('/*', function(req, res) {
res.sendFile(path.join(__dirname, 'public/index.html'));
});
// Start server
app.listen(PORT, function() {
console.log('Server ' + cluster.worker.id + ' running on ' + PORT);
});
}
Handling the import:
Great question, from my experience by far the fastest way to insert a csv into mongo is via the command line:
mongoimport -d db_name -c collection_name --type csv --file file.csv --headerline
I don't believe mongoose has a way of calling mongoimport (someone correct me if I'm wrong)
But it's simple enough to call via node directly:
var exec = require('child_process').exec;
var cmd = 'mongoimport -d db_name -c collection_name --type csv --file file.csv --headerline';
exec(cmd, function(error, stdout, stderr) {
// do whatever you need during the callback
});
The above will have to be modified to be dynamic, but it should be self-explanatory.
Handling the upload:
Uploading the file from a front-end client is another challenge.
Most browsers will timeout if you make a request to a server and don't get a response within 60 seconds (probably what you are referring to above)
One solution would be to open a socket connection (search for socket.io in npm) for details. This will create a constant connection to the server and won't be subject to the timeout restrictions.
If uploading is not an issue, and the timeout is due to the slow parsing/inserting then you may not have to worry about this once you implement the above.
Other considerations:
I'm not sure exactly what you need to send back to the user, or what parsing needs to take place. But that can either be done outside of the normal request/response cycle, or can be handled during a socket connection if it's needed during one request/response cycle.
I'm new to node and got stuck with handling multiple async tasks.
Except from node, I've got another server (S1) which doesn't return data immediately to requests, it can returns multiple types of data and also can send notifications without requesting them specifically, so node have to listen to data from it , parse it and act accordingly.
The connection to this server (S1) is done by using:
S1 = net.createConnection({'host':S1Host, 'port': S1Port});
And node listens to data with:
S1.on('data', function(data){
S1DataParse(data);
});
I have to route the correct data (after parsing it) to a specific POST request.
app.post('/GetFooFromS1', function(req, res){
// Send request to S1
S1.write({'type':'foo'});
// If got the correct data sometime in the future, send response to the browser
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':S1FooData});
});
I tried to use the async module for that, but with no success.
What I was trying to do:
var asyncTasks = [];
app.post('/GetFooFromS1', function(req, res){
asyncTasks.push(function(callback){
// Send request to S1
S1.write({'type':'foo'});
});
async.parallel(asyncTasks, function(response){
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':response});
});
});
and another task in S1DataParse:
function S1DataParse(){
if(data.type='foo'){
asyncTasks.push(function(callback){
callback(data);
});
}
}
But, of course, the second task never added to the asyncTasks array. I really got stuck with that.
Can you please help me with that?
Thanks
-=-=-=- Edit -=-=-=-
Eventually, I came accross with events and EventEmitter().
From the POST request I call the function that sends requests to the data server (DataServerClientGet).
In this function I register a listener which will get the future data.
eventEmitter.on('getData', returnDataServerData);
It all works great except for one thing. Whenever I refresh the page or add other POST requests, I get an error:
Error: Can't set headers after they are sent.
It would be great if I solve this problem. Help me, please.
Thanks ;)
The whole code looks like this:
var express = require('express');
var app = express();
var http = require('http').Server(app);
var bodyParser = require('body-parser')
var net = require('net');
var events = require('events');
var dataServerHost = '127.0.0.1';
var dataServerPort = 12345;
var dataServerClient;
var logMsg;
var eventEmitter = new events.EventEmitter();
/*******************************************/
// Init
/*******************************************/
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
app.use(express.static(__dirname + '/public'));
/*******************************************/
// Connect to the data server
/*******************************************/
DataServerConnect();
/*******************************************/
// Open listener on port 3000 (to browser)
/*******************************************/
http.listen(3000, function(){
logMsg = 'listening on *:3000';
console.log(logMsg);
});
/*******************************************/
// Routing
/*******************************************/
app.get('/', function(req, res){
res.sendFile(__dirname + '/index.html');
});
app.post('/GetDataFoo', function(req, res){
var msg;
var size;
msg ='\n{"Type":"Query", "SubType":"GetDataFoo","SearchFilter":""}';
size = msg.length;
logMsg = 'Client to DataServer: GetDataFoo';
console.log(logMsg);
DataServerClientGet('GetDataFoo', size, msg, res);
});
/*******************************************/
// Functions
/*******************************************/
function DataServerConnect(){
dataServerClient = net.createConnection({'host':dataServerHost, 'port': dataServerPort}, function(){
logMsg = 'Connected to DataServer ['+dataServerHost+':'+dataServerPort+']';
console.log(logMsg);
});
dataServerClient.on('data', function(data){
logMsg = 'DataServerData>>>\n'+data.toString()+'DataServerData<<<';
console.log(logMsg);
DataServerDataParse(data.toString());
});
dataServerClient.on('end', function(){
logMsg = 'Disconnected from DataServer';
console.log(logMsg);
});
}
function DataServerClientGet(type, size, msg, res){
dataServerClient.write('Type: Json\nSize: '+size+'\n\n'+msg, function(err){
var returnDataServerData = function returnDataServerData(results){
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':results});
}
eventEmitter.on('getData', returnDataServerData);
}
function DataServerDataParse(json){
if(json.Type=='GetDataFoo')
{
var MessageList = json.MessageList;
eventEmitter.emit('getData', MessageList);
}
}
-=-=-=- Edit -=-=-=-
The Error: Can't set headers after they are sent. caused by adding the same listener of the same type each time the DataServerClientGet was called and the res was sending multiple times.
I solved this one by adding: removeListener(event, listener)
right after the res, inside the function. Anyway, I think it's wrong and can cause problems if there will be multiple calling to DataServerClientGet with the same type etc.
There is a optional callback parameter that you can pass to write function(docs), something like :
S1.write({'type':'foo'},function(err){
if(err){
//Handle error
}else{
res.setHeader('Content-Type', 'application/json');
res.json({'status':'success', 'value':response});
}
})
This can work with post route , but in your 'data' listener ,you cant send data from server to client when there is not connection initialized by client (it is not bidireccional ) if you want bidireccional behavior you can check socket.io
I have a file in D: Drive of my local system. I need to download the file into the E: Drive. How to do this using node.js and http request? I am a beginner in node.js. Please give me valuable suggestions.
Note: The file may be in any type.
Here is an example:
// upload.js
var fs = require('fs')
var newPath = "E:\\newPath";
var oldPath = "D:\\oldPath";
exports.uploadFile = function (req, res) {
fs.readFile(oldPath, function(err, data) {
fs.writeFile(newPath, data, function(err) {
fs.unlink(oldPath, function(){
if(err) throw err;
res.send("File uploaded to: " + newPath);
});
});
});
};
// app.js
var express = require('express'), // fetch express js library
upload = require('./upload'); // fetch upload.js you have just written
var app = express();
app.get('/upload', upload.uploadFile);
app.listen(3000);
Basically there are two parts, one doing the copying from one drive to another, and the other one is for triggering. Once you run you app.js and make a GET request to localhost:3000/upload it will copy the file from newPath to the oldPath. For further information have a look to expressjs and fs.
Assuming it's a text file, you would have to write two node.js server.
The first would answer (all/specific, your choice) http get with the content of the file, the other would make a get and download the file.
server.js: Will work only for text file
var http = require('http'),
fs = require('fs'),
server = http.createServer(function (req, res){
res.writeHead(200, {'content-type': 'text/text'});
fs.readFile('E:/path/to/file.txt', function (data) {
res.write('' + data);
res.end();
});
}).listen(8080);
client.js
var http = require('http'),
fs = require('fs'),
file = fs.createWriteStream('D:/path/to/new.txt', {flags: 'w'});
http.get('http://localhost:8080', function (res) {
res.pipe(file, {end: 'false'});
res.on('end', function() {
file.end();
});
});
EDIT:
The only advantage versus anvarik's solution is that I don t use express...