nodejs tcp file upload missing parts - node.js

I am trying to make a server and client program just on localhost to be able to upload and list files from the server. I have the code to upload files to the server but for some reason after the server writes the file from the client it has less bytes than the original file.
Here is the client code:
var net = require('net');
var fs = require('fs');
var port='3000';
var host='localhost';
var path=__dirname;
var fileUploadDir = __dirname+'/uploadfolder/';
process.stdin.resume();
var socket = net.connect(port,host);
var stdin = process.stdin;
var stdout = process.stdout;
stdout.on('data', function(d){
console.log("d: "+d);
});
socket.on('connect',function(){
console.log("connected to server");
//stdin.pipe(stdout);
stdin.pipe(socket, {end: false});
});
socket.on('data', function(data){
var b = new Buffer(data, "utf8");
var bStr = b.toString();
if(bStr=="okupload"){
console.log("uploading file");
folderFilesUpload(socket);
}else{
console.log(bStr);
}
});
socket.on('end', function(){
socket = net.connect(port,host);
console.log('socket ended');
});
function folderFilesUpload(socket){
fs.readdir(fileUploadDir, function(err, results){
if(err)
console.log("upload to upload files");
else{
if(results.length > 0){
for(var r in results){
var fullPath = fileUploadDir+results[r];
var fileStream = fs.createReadStream(fullPath);
fileStream.on('open', function(){
fileStream.pipe(socket);
});
}
}else{
console.log("no files to upload");
}
}
});
}
Server Code:
var net=require('net');
var fs=require('fs');
var host ='localhost';
var port='3000';
var outputDir = __dirname+'/outputdir/';
var uploadState = false;
var server = net.createServer(function(conn){
console.log('server connected');
conn.on('data', function(data) {
if(data == "l\n" && !uploadState){
listFiles(conn);
}
if(data == "u\n"){
console.log("switching to upload file state");
uploadState=true;
conn.write("okupload");
}
if(uploadState){
console.log("data");
//console.log("data: "+data);
var file = fs.createWriteStream(outputDir+'temp');
conn.pipe(file);
}
});
conn.on('end', function(){
uploadState = false;
console.log('data has ended');
});
});
server.listen(port,host, function(){
server.on('connection', function(){
console.log('connection made');
});
});
function listFiles(conn){
console.log("listing files");
fs.readdir(outputDir, function(err, results){
if(err){
var b = new Buffer("error listing files", "utf8");
conn.write(b);
}else{
var output = "files: \n";
if(results !== null && results.length >0){
for(var r in results){
output += results[r]+"\n";
}
}else{
output += "no files.\n";
}
var b = new Buffer(output, "utf8");
conn.write(b);
}
});
}

Related

Node js Printing info from JSON file using a function from another file V2.0

This is a continuation from another question I asked earlier Node.js Printing info from JSON file using a function from another JS file
In my previous question I had problems in calling a function from my data-service.js file that printed all the items in my JSON array, and had it resolved, but now I'm struggling in doing something similar in printing only the employees from my JSON array that I specify through the url. For example http://localhost:8080/employeesstatus=5 would print only the employee with a status of 5 however nothing is getting printed
SERVER.JS
var HTTP_PORT = process.env.PORT || 8080;
var express = require('express');
var data = require('./data-service');
var fs = require('fs');
var app = express();
var object = require('./data-service');
console.log("Express http server listening on 8080");
//app.get('/employees', function(req,res){
// return object.getAllEmployees()
// .then((response) => res.send(response))
//}); //QUESION FROM PREVIOUS POST WHICH WAS RESOLVED
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
DATA SERVICE.JS
var employees = [];
var departments = [];
var error = 0;
var fs = require("fs");
function initialize(){
employees = fs.readFileSync("./data/employees.json", 'utf8', function(err, data){
if(err){
error = 1;
}
employees = JSON.parse(data);
});
departments = fs.readFileSync("./data/department.json", 'utf8',
function(err, data){
if(err){
error = 1;
}
departments = JSON.parse(data);
});
}
function check() {
return new Promise(function(resolve,reject){
if (error === 0){
resolve("Success");
}
else if(error === 1){
reject("unable to read file");
}
})
};
var getAllEmployees = function(){
return check().then(function(x){
console.log(x);
console.log(employees);
return employees;
}).catch(function(x){
console.log("No results returned");
});
}
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return console.log(employees2[i]);
}
}
}).catch(function(){
console.log("no results returned");
})
}
module.exports.getAllEmployees = getAllEmployees;
module.exports.getEmployeesByStatus = getEmployeesByStatus;
The 2 functions in question
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return employees2[i];
}
}
}).catch(function(){
console.log("no results returned");
})
}
1) You should replace /employees route with the following
app.get('/employees/status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
You are able to access using http://localhost:8080/employees/status=5
2) Return employees2[i] instead of console.log(employees2[i]).

SFTP Server for reading directory using node js

I have created a node based SSH2 SFTP Server and Client. My objective is to read directory structure of SFTP Server. Suppose I have an SFTP Server containing folder temp, I want to read files inside the temp directory. I am using ssh2 npm module for creating SFTP Server and Client. It is making connection to SFTP Server but not listing the directory
Below is the code
CLIENT SIDE SFTP
var Client = require('ssh2').Client;
var connSettings = {
host: 'localhost',
port: 22,
username:'ankit',
password:'shruti',
method:'password'
// You can use a key file too, read the ssh2 documentation
};
var conn = new Client();
conn.on('ready', function() {
console.log("connected to sftp server")
conn.sftp(function(err, sftp) {
if (err)
throw err;
sftp.readdir('',function(err,list)
{
console.log('Inside read')
if(err)
{
console.log(err);
throw err;
}
console.log('showing directory listing')
console.log(list);
conn.end();
})
/**
var moveFrom = "/remote/file/path/file.txt";
var moveTo = __dirname+'file1.txt';
sftp.fastGet(moveFrom, moveTo , {}, function(downloadError){
if(downloadError) throw downloadError;
console.log("Succesfully uploaded");
});
*/
})
}).connect(connSettings);
SERVER SIDE SFTP :
var constants = require('constants');
var fs = require('fs');
var ssh2 = require('ssh2');
var OPEN_MODE = ssh2.SFTP_OPEN_MODE;
var STATUS_CODE = ssh2.SFTP_STATUS_CODE;
var srv = new ssh2.Server({
hostKeys: [fs.readFileSync(__dirname+'/key/id_rsa')],debug:console.log
}, function(client) {
console.log('Client connected!');
client.on('authentication', function(ctx) {
if (ctx.method === 'password'
// Note: Don't do this in production code, see
// https://www.brendanlong.com/timing-attacks-and-usernames.html
// In node v6.0.0+, you can use `crypto.timingSafeEqual()` to safely
// compare two values.
&& ctx.username === 'ankit'
&& ctx.password === 'shruti')
ctx.accept();
else
ctx.reject();
}).on('ready', function() {
console.log('Client authenticated!');
});
client.on('session',function(accept,reject)
{
console.log("Client asking for session");
var session = accept();
var handleCount = 0;
var names=[]
session.on('sftp',function(accept,reject)
{
console.log('Client sftp connection')
var sftpStream = accept();
sftpStream.on('OPENDIR',function(reqID,path)
{
var handle = new Buffer(4);
handle.writeUInt32BE(handleCount++, 0, true);
sftpStream.handle(reqID,handle);
console.log(handle);
console.log('Opening Read Directory for --'+path);
console.log(reqID);
console.log(path);
}).on('READDIR',function(reqID,handle)
{
console.log('Read request')
console.log(reqID);
if(handle.length!==4)
return sftpStream.status(reqID, STATUS_CODE.FAILURE,'There was failure')
sftpStream.status(reqID, STATUS_CODE.OK,'Everything ok')
sftpStream.name(reqID,names);
})
})
})
}).listen(0, '127.0.0.1', function() {
console.log('Listening on port ' + this.address().port);
});
srv.listen(22)
The client will send multiple READDIR requests. On the first request, you should send the file listing, on the second you should send an EOF status, to tell the client the listing has finished.
An example would be:
let listSent = false;
let names = [];
sftpStream.on('OPENDIR', function (reqID, path) {
listSent = false;
sftpStream.handle(new Buffer());
});
sftpStream.on('READDIR', function (reqID, handle) {
if (listSent) {
sftpStream.status(reqID, STATUS_CODE.EOF);
return;
}
sftpStream.name(reqID, names);
listSent = true;
});

Nodejs - data transfer between server and client

I was given a task to send JSON string from client to server and from server to client, whenever there is a new record found to send.
I decided to build TCP connection(suggest me if there is any other better way in Node.js) between server and client to transfer data.
The problem is, I was supposed to use a delimiter to separate JSON strings one from another. I am afraid what if the json string contains the delimiter string inside the object. I am looking for a better way to separate two JSON strings.
Below is my code. Please help me.
Client
var net = require('net')
, client = new net.Socket();
var chunk = ''
, dlim_index = -1
, delimit = '~~';
client.connect(config.Port, config.IpAddress, function () {
console.log('Server Connected');
client.write('CLIENTID:' + process.argv[2]);
client.write(delimit);
});
client.on('data', function (data) {
var recvData = data.toString().trim();
chunk += recvData;
dlim_index = chunk.indexOf(recvData);
console.log(data);
while (dlim_index > -1) {
var useData = chunk.substring(0, dlim_index);
if (useData == 'SUCCESS') {
controller.listenOutQueue(function (dataToSend) {
var object = JSON.parse(dataToSend);
client.write(dataToSend);
client.write(delimit);
});
}
else {
var record = JSON.parse(useData);
controller.insertIntoQueue(record, function (status) {
});
}
chunk = chunk.substring(dlim_index + 2);
dlim_index = chunk.indexOf(delimit);
}
});
client.on('close', function () {
console.log('Connection closed');
});
client.setTimeout(50000, function () {
//client.destroy();
});
Server
var net = require('net')
, server = net.createServer()
, delimit = '~~'
, clients = [];
controller.listenOutQueue(function (dataToSend) {
client.write(dataToSend);
client.write(delimit);
});
server.on('connection', function (socket) {
var chunk = '';
var dlim_index = -1;
socket.on('data', function (data) {
var recvData = data.toString().trim();
chunk += recvData;
dlim_index = chunk.indexOf(delimit);
while (dlim_index > -1) {
var useData = chunk.substring(0, dlim_index);
if (useData.substring(0, 9) == 'CLIENTID:') {
socket.clientid = useData.replace('CLIENTID:', '');
console.log('Client Id: ' + socket.clientid);
clients.push(socket);
var successMessage = "SUCCESS";
socket.write(successMessage);
socket.write(delimit);
}
else {
controller.insertIntoQueue(JSON.parse(useData), function (status) {
});
}
chunk = chunk.substring(dlim_index + 2);
dlim_index = chunk.indexOf(delimit);
}
});
socket.on('end', function () {
console.log('Connection Closed (' + socket.clientid + ')');
});
socket.on('error', function (err) {
console.log('SOCKET ERROR:', err);
});
});
server.listen(config.Port, config.IpAddress);

Transferring file net

I would like to transfer files between one client, and a server.
I only use stream, and fs modules.
For the moment, I can transfer ".txt" files perfectly, but it does not work for ".mp3" files (for instance). Indeed, the size of the ".mp3" file saved from the server has a bigger size than the original file. (eg : 9,2 mo instead of 5mo) and I can not open it.
Here is my server part:
var net = require('net');
var util = require("util");
var fs = require('fs');
var receive_object = {};
var server = net.createServer(function (socket){
console.log('server connected');
socket.write('hello\r\n');
//socket.pipe(socket);
socket.on('data',function(data){
if(data.toString() == "upload_demand"){
socket.write("upload_demand_ok");
}else if(data.toString().indexOf("fileName")>-1){
data = data.toString().replace("fileName","");
fs.exists("server_"+data, function(exists){
if(exists) socket.write("fileName_ko");
else{
receive_object.fileName = data;
receive_object.writable = fs.createWriteStream("server_"+receive_object.fileName);
receive_object.contenu = "";
socket.write("fileName_ok");
}
});
}else if(data.toString().indexOf("size")>-1){
data = data.toString().replace("size","");
receive_object.size_total = data;
receive_object.size_received = 0;
socket.write("size_ok");
}/*else if(data.toString()=="upload_success"){
receive_object.writable.write(receive_object.contenu);
console.log('It\'s saved!');
}*/
else{
receive_object.size_received += 64*1024;
socket.write("uploading"+parseInt(receive_object.size_received/receive_object.size_total*100));
receive_object.contenu += data;
}
});
socket.on('end', function() {
console.log('client disconnected');
receive_object.writable.write(receive_object.contenu);
});//socket on
});
/**
let's listening the server
*/
server.listen(1337, '127.0.0.1',function(){
console.log("server bound on port %j and on address %j",server.address().port,server.address().address);
});
Here is my client part:
var util = require("util");
var net = require("net");
var fs = require('fs');
var fileName = "exemple.txt";
/**
client connecting to the corresponding port Server
*/
var client = net.connect({port: 1337}, function () {
util.log("Client connected");
fs.exists(fileName, function(exists) {
if (exists) {
fs.stat(fileName, function(error, stats) {
transfer_object = {};
transfer_object.fileName = fileName;
transfer_object.size = stats.size;
transfer_object.stream = fs.createReadStream(fileName, { bufferSize: 64 * 1024 });
transfer_object.progression = 0;
client.write("upload_demand");
});
}else{
console.log("fileNotFound");
}
});
});
client.on('data', function(data) {
if(data=="upload_demand_ok"){
console.log(''+data);
client.write("fileName"+transfer_object.fileName);
}
else if(data == "fileName_ok"){
console.log(''+data);
client.write("size"+transfer_object.size)
}else if(data == "fileName_ko"){
console.log('ficher deja existant');
}else if(data == "size_ok"){
console.log(''+data);
transfer_object.stream.pipe(client,{ end: true });
}else if(data.toString().indexOf("uploading")>-1){
data = parseInt(data.toString().replace("uploading",""));
progressbar(data);
//if(data>=100) client.write('upload_success');
}
});
/**
client receive end event from server
*/
client.on('end', function() {
console.log('client disconnected');
});
function progressbar(int){
if(int>=0 && int<=20 && transfer_object.progression == 0){
transfer_object.progression = 1;
process.stdout.write("progression : [=");
}
else if(int>20 && int<=40 && transfer_object.progression == 1){
transfer_object.progression = 2;
process.stdout.write("==");
}else if(int>40 && int<=60 && transfer_object.progression == 2){
transfer_object.progression = 3;
process.stdout.write("==");
}else if(int>60 && int<=80 && transfer_object.progression == 3){
transfer_object.progression = 4;
process.stdout.write("==");
}else if(int>80 && int<=100 && transfer_object.progression == 4){
transfer_object.progression = 5;
process.stdout.write("=]");
}
}
The txt file has utf8 encoding, and the mp3 - binary encoding.
You can set the encoding usable for stream as described here http://nodejs.org/api/stream.html#stream_writable_write_chunk_encoding_callback
According to http://nodejs.org/api/buffer.html#buffer_buffer
The utf8 is for txt, and binary for mp3 data (but manual do not recommend using it)

Get my twitter timeline instead of public twitter timeline using Node JS

This code works fine but it seems that it retrieve any tweet not only the people I follow, I want to get my timeline tweets
what should I do ?
var open = require('open');
var sys = require('sys');
var spawn = require('child_process').spawn
var OAuth= require('oauth').OAuth;
var express = require("express");
var util = require('./util');
var app = express();
var oa = new OAuth(
"https://api.twitter.com/oauth/request_token",
"https://api.twitter.com/oauth/access_token",
"NeFpQZNDapDtFJnUXDe0Q",
"mXf7AKYJWIdiToKuSgsTm2me0GBA1aNn39uskmEoOM",
"1.0",
"http://127.0.0.1:8080/twitterCallback",
"HMAC-SHA1"
);
var oauthtoken ,oauthtokensecret;
{
oa.getOAuthRequestToken(function(error, oauth_token, oauth_token_secret, results){
if (error) {
console.log(error);
res.send("yeah no. didn't work.")
}
else {
oauthtoken = oauth_token;
oauthtokensecret = oauth_token_secret;
open('http://twitter.com/oauth/authenticate?oauth_token='+oauthtoken);
}
});
}
var http = require('http');
var fs = require('fs');
function readFile(fileName,response){
fs.readFile(fileName, function (err,data) {
if (err) {
return console.log(err);
}
response.end(data);
});
}
var server = http.createServer(function(req, res) {
if(req.url == '/favicon.ico'){
res.end('Hello Http');
}else if(req.url.indexOf('twitterCallback') != -1){
res.writeHead(200);
res.end('Hello Http');
var url = require('url');
var url_parts = url.parse(req.url, true);
var query = url_parts.query;
var oauth_verifier = query.oauth_verifier;
console.log(oauth_verifier);
oa.getOAuthAccessToken(oauthtoken,oauthtokensecret,query.oauth_verifier, function(error, oauth_access_token, oauth_access_token_secret, results)
{
if (error){
console.log(error);
} else {
//console.log("oauth_access_token = " + oauth_access_token);
//console.log("oauth_access_token_secret = " + oauth_access_token_secret);
}
var twitter = require('ntwitter');
var twit = new twitter({
consumer_key: 'NeFpQZNDapDtFJnUXDe0Q',
consumer_secret: 'mXf7AKYJWIdiToKuSgsTm2me0GBA1aNn39uskmEoOM',
access_token_key: oauth_access_token,
access_token_secret: oauth_access_token_secret
});
twit.stream('statuses/sample', function(stream) {
stream.on('data', function (data) {
console.log(data.text);
});
});
}
);
}else{
readFile(req.url.substr(1));
}
});
server.listen(8080);
Use the ntwitter library : https://github.com/AvianFlu/ntwitter
And to consume api :
var twitter = require('ntwitter');
var twit = new twitter({
consumer_key: 'Twitter',
consumer_secret: 'API',
access_token_key: 'keys',
access_token_secret: 'go here'
});
twit.stream('user', {track:'nodejs'}, function(stream) {
stream.on('data', function (data) {
console.log(data);
});
stream.on('end', function (response) {
// Handle a disconnection
});
stream.on('destroy', function (response) {
// Handle a 'silent' disconnection from Twitter, no end/error event fired
});
// Disconnect stream after five seconds
setTimeout(stream.destroy, 5000);
});

Resources