I am creating an app using Node and react as part of my learning process. The which covert HTTP, https links to torrent. The problem is, I need to download the file first to convert it to torrent. The app working fine when it comes to small files. But if it is a large file like One GB it fails and app crash.
here is my project link:-
https://github.com/savadks95/FireBit/blob/master/server.js
Can anyone please tell me how to fix this. And I also want to show the download in progress.
var http = require("http");
var webtorrentify = require("webtorrentify-link");
var fs = require("fs");
var url = require("url");
var path = require("path");
var validUrl = require("valid-url");
var express = require("express");
var getUrls = require("get-urls");
var remote = require("remote-file-size");
var app = express();
var downloadLink;
var fileName;
var fileSize;
var server;
var parsed;
var param;
var link;
var port;
port = process.env.PORT || 80;
app.get("/favicon.ico", function(req, res) {
console.log("favicon request recived");
});
app.get("*", function(req, res) {
if (req.url === "/") {
//app.use('/public/html', express.static(path.join(__dirname)));
fs.readFile("public/html/index.html", function(err, data) {
res.write(data);
});
} else if (req.url === "/l?thelink=") {
fs.readFile("public/html/emptyRequest.html", function(err, data) {
res.write(data);
res.end();
});
} else {
//---------Reciving Url--------------------
console.log(req.query.thelink);
downloadLink = req.query.thelink;
//-----------------------------------------
//------------checking for valid url-------
if (validUrl.isUri(downloadLink)) {
console.log("Looks like an URI");
//-----------------------------------------
//----------Extracting filename-------------
parsed = url.parse(downloadLink);
fileName = path.basename(parsed.pathname);
console.log(path.basename(parsed.pathname));
//-------------------------------------------
//----------Finding File size----------------
remote(downloadLink, function(err, o) {
fileSize = o / 1024 / 1024;
console.log("size of " + fileName + " = " + fileSize + " MB");
//-------------------------------------------
if (fileSize < 501) {
///////////////Creating Torrent////////////////////
webtorrentify(downloadLink).then(function(buffer) {
console.log("creating the torrent");
//res.send('what is');
//-------------------------------------------
res.setHeader("Content-Type", "application/x-bittorrent");
res.setHeader(
"Content-Disposition",
`inline; filename="${fileName}.torrent"`
);
res.setHeader("Cache-Control", "public, max-age=2592000"); // 30 days
res.send(buffer);
console.log(fileName + ".torrent created");
res.end();
//-------------------------------------------
});
////////////////////////////////////////////////
} else {
console.log("More than 500 MB");
res.send("<h4> More than 500 MB or invalid URL </h4>");
}
});
} else {
console.log("not url");
fs.readFile("public/html/404.html", function(err, data) {
res.write(data);
res.end();
});
}
}
});
app.listen(port);
Related
I just set up my first node HTTP server, and I am trying to get the response data from a JSON file in my application. When I declare a JSON object in the server.js file, all works well.
data = "{"sample json" : "this is a test"}";
But I want to replace data with a static JSON file at data/sample.json
Here's an example in my server.js file
const http = require("http");
const hostname = "localhost";
const port = 3000;
const server = http.createServer(function(req, res) {
data = // this is where I want to get the JSON data from data/sample.json
res.writeHead(200, {'Content-Type': 'application/json'});
res.write(data);
res.end();
});
Solved with fs.readFile()
const http = require("http");
const hostname = "localhost";
const port = 3000;
const fs = require('fs');
const server = http.createServer(function(req, res) {
filePath = './data/sample.json';
if (req.path == '/data/sample.json') {
fs.readFile(filePath, function(error, content) {
if (error) {
if (error.code == 'ENOENT') {
response.writeHead(404);
response.end(error.code);
}
else {
response.writeHead(500);
response.end(error.code);
}
}
else {
res.writeHead(200, {'Content-Type': 'application/json'});
res.end(content);
}
});
}
else {
response.writeHead(404);
response.end('restricted path');
}
});
In case someone else tumbles upon this question. The answer above has ton of typos and errors and incomplete. Here is a working solution.
const http = require("http");
const hostname = "localhost";
const fs = require('fs');
const port = 8080;
const server = http.createServer(function(req, res) {
filePath = './data/sample.json';
if (req.url == '/api/data') {
fs.readFile(filePath, function(error, content) {
if (error) {
if (error.code == 'ENOENT') {
res.writeHead(404);
res.end(error.code);
}
else {
res.writeHead(500);
res.end(error.code);
}
}
else {
res.writeHead(200, {'Content-Type': 'application/json'});
res.end(content);
}
});
}
else {
res.writeHead(404);
res.end('404 NOT FOUND');
}
});
server.listen(port, hostname, () => {
console.log('Server started on port ', port);
});
I have a node server running on port 3000 to serve api request.
this works fine...
http://www.skoolaide.com:3000/api/accounts
this does not.
https://www.skoolaide.com:3000/api/accounts
Can someone tell me why? If you go to https://www.skoolaide.com, the ssl cert is configured correctly, but do I need to do something on the node side?
this is my server js file. Please note: is only runs the middleware(api) on port 3000. For some reason I cannot access the middleware via https...
'use strict';
var express = require('express');
var router = express.Router();
var app = express();
var http = require('http');
var open = require('open');
var cors = require('cors');
var path = require('path');
var morgan = require('morgan');
var errorhandler = require('errorhandler');
var bodyParser = require('body-parser');
var mongoose = require('mongoose');
var config = require('./config/config');
var jwt = require('jsonwebtoken');
var compression = require('compression');
var runMiddleware = require('run-middleware')(app);
var fs = require('fs');
var readline = require('readline');
var google = require('googleapis');
var googleAuth = require('google-auth-library');
var multer = require('multer');
var node_xj = require("xls-to-json");
var moment = require('moment');
var async = require('async');
var btoa = require('btoa');
var sharp = require('sharp');
var students = require("./middleware/students.api");
var accounts = require("./middleware/accounts.api");
var messages = require("./middleware/messages.api");
var advocates = require("./middleware/advocates.api");
var authenticate = require("./middleware/authenticate.api");
var email = require("./middleware/email.api");
var text = require("./middleware/text.api");
var colleges = require("./middleware/colleges.api");
var amazon = require("./middleware/amazon.api");
var rewards = require("./middleware/rewards.api");
var files = require("./middleware/files.api");
var validations = require("./middleware/validations.api");
var points = require("./middleware/points.api");
var notifications = require("./middleware/notifications.api");
var notificationsMap = require("./middleware/notificationsMap.api");
var trivia = require("./middleware/trivia.api");
var tasks = require("./middleware/rewardgoals.api");
var classes = require("./middleware/classes.api");
var connections = require("./middleware/connections.api");
var badges = require("./middleware/badges.api");
var fixpasswords = require("./middleware/fixpasswords.api");
var Files = require('./models/files');
mongoose.connect(config.database);
process.on('SIGINT', function() {
mongoose.connection.close(function () {
console.log('Mongoose disconnected on app termination');
process.exit(0);
});
});
// use body parser so we can get info from POST and/or URL parameters
app.use(bodyParser.json({
limit: '50mb'
}));
app.use(bodyParser.urlencoded({
limit: '50mb',
extended: true
}));
app.set('uploads', path.join(__dirname, 'uploads'));
app.get("/uploads/*", function (req, res, next) {
res.sendFile(__dirname + req.url);
});
var whitelist = ['https://www.skoolaide.com', 'https://skoolaide.com'];
var corsOptionsDelegate = function (req, callback) {
var corsOptions;
if (whitelist.indexOf(req.headers['origin']) !== -1) {
corsOptions = { origin: true, credentials: true } // reflect (enable) the requested origin in the CORS response
} else {
corsOptions = { origin: false, credentials: false } // disable CORS for this request
}
callback(null, corsOptions) // callback expects two parameters: error and options
}
app.use(cors(corsOptionsDelegate));
//this is used because the body parser removes undefined values from the database.
app.set('json replacer', function (key, value) {
// undefined values are set to `null`
if (typeof value === "undefined") {
return null;
}
return value;
});
app.use(multer({
dest: './uploads/',
rename: function (fieldname, filename) {
return filename.replace(/\W+/g, '-').toLowerCase() + Date.now()
},
onFileUploadStart: function (file) {
//console.log(file.fieldname + ' is starting ...')
},
onFileUploadData: function (file, data) {
//console.log(data.length + ' of ' + file.fieldname + ' arrived')
},
onFileUploadComplete: function (file) {
//console.log(file.fieldname + ' uploaded to ' + file.path)
}
}).any());
// "files" should be the same name as what's coming from the field name on the client side.
app.post("/api/upload", function (req, res) {
//console.log(req.files[0].mimetype)
fs.readFile(req.files[0].path, function (err, data) {
var newPath = __dirname + "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
var readPath = "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
if(req.files[0].mimetype.indexOf('image') > -1){
sharp(data)
.rotate()
.resize(800, 800)
.max()
.toFile(newPath, function(err, info){
var file = new Files();
file.owner = req.headers.account_id || null;
file.classId = req.body.classId || null;
file.rewardGoalId = req.body.rewardGoalId || null;
file.avatarId = req.body.avatarId || null;
file.mimeType = req.files[0].mimetype || null;
file.originalName = req.files[0].originalname || null;
file.path = readPath;
file.save(function (err, newFile) {
if (err) {
res.send(err);
} else {
res.send({ success: true, data: newFile }).end();
}
});
});
} else{
//not an image file...
var newPath = __dirname + "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
//console.log('Writing file: ', newPath);
fs.writeFile(newPath, data, function (err) {
var readPath = "/uploads/" + req.headers["account_id"] + '_' + moment().format('MM_DD_YYYY_HH-mm-ss') + '_' + req.files[0].originalname.replace(/[^a-zA-Z0-9.]/g, '_');
//console.log(readPath)
var file = new Files();
file.owner = req.headers.account_id || null;
file.classId = req.body.classId || null;
file.rewardGoalId = req.body.rewardGoalId || null;
file.profileId = req.body.profileId || null;
file.mimeType = req.files[0].mimetype || null;
file.originalName = req.files[0].originalname || null;
file.path = readPath;
file.save(function (err, newFile) {
if (err) {
res.send(err);
} else {
res.send({ success: true, data: newFile }).end();
}
});
});
}
});
});
app.use(express.static('www'))
var a = ['/'];
//all get requests resolve to index.
app.get(a, function (req, res) {
console.log('This is a test', req.originalUrl, req.url);
res.setHeader('Cache-Control', 'private, no-cache, no-store, must-revalidate');
res.setHeader('Expires', '-1');
res.setHeader('Pragma', 'no-cache');
res.setHeader('X-UA-Compatible', 'IE=Edge,chrome=1');
res.setHeader('Judson', 'Rocks!');
if (req.originalUrl == '/') {
res.sendFile('www/index.html', {root: __dirname});
}
});
app.set("superSecret", config.secret)
//var upload = require("./middleware/upload.api");
app.use('/api', router);
app.use('/api/students', students);
app.use('/api/accounts', accounts);
app.use('/api/messages', messages);
app.use('/api/advocates', advocates);
app.use('/api/authenticate', authenticate);
app.use('/api/email', email);
app.use('/api/text', text);
app.use('/api/colleges', colleges);
app.use('/api/amazon', amazon);
app.use('/api/rewards', rewards);
app.use('/api/files', files);
app.use('/api/validate', validations);
app.use('/api/points', points);
app.use('/api/notifications', notifications);
app.use('/api/notificationsMap', notificationsMap);
app.use('/api/trivia', trivia);
app.use('/api/rewardgoals', tasks);
app.use('/api/classes', classes);
app.use('/api/badges', badges);
app.use('/api/connections', connections);
app.use('/api/fixpasswords', fixpasswords);
/**SERVER*************************************/
// all environments
app.set('port', process.env.PORT || 3000);
app.engine('html', require('ejs').renderFile);
// express/connect middleware
app.use(morgan('dev'));
app.use(compression()); //use compression
// development only
if ('development' === app.get('env')) {
app.use(errorhandler());
}
/**END SERVER*************************************/
var cluster = require('cluster');
if (cluster.isMaster) {
var numWorkers = require('os').cpus().length;
console.log('Master cluster setting up ' + numWorkers + ' workers...');
for (var i = 0; i < numWorkers; i++) {
cluster.fork();
}
cluster.on('online', function (worker) {
console.log('Worker ' + worker.process.pid + ' is online');
});
cluster.on('exit', function (worker, code, signal) {
console.log('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
console.log('Starting a new worker');
cluster.fork();
});
} else {
app.listen(app.get('port'),
function () {
console.log('myApp server listening on port ' + app.get('port'));
});
}
Your code only starts an HTTP server. This line:
app.listen(app.get('port'))
starts only an http server. If you want support for HTTPS, you have to also start an HTTPS server and you have to start it on a different port. The express doc for app.listen() shows you how to do that and it's basicaly like this:
var express = require('express');
var https = require('https');
var http = require('http');
var app = express();
http.createServer(app).listen(80);
https.createServer(options, app).listen(443);
where options contains your SSL certificates and where you fill in the desired port numbers for your http server and your https server.
If you were operating under the illusion that you don't need a port number in the browser in order to access both an http and https server, that's because the browser fills in a default port number for you. If you just type an http URL, it uses port 80. If you type an https URL with no port, it uses port 443. If you're not on those two specific ports, you have to specify the desired port in the browser URL. In either case, you need a separate server on separate ports for both http and https.
Use var https = require("https"); not
Var http = require ("http");
I'm having some problems when trying to stream videos in nodejs. When I tried just pass the video path to the source/video html tag, it doenst worked. Then, I realized that problably I would have to stream the video.
The problem is: When I stream the video, I just get the video being played in the browser, as a direct link to some downloaded video, not the renderized page with some data(video title and path).
I wanna render the page and then run the video.
When I render, I receive the error: "Can't set headers after they're sent".
My code:
const express = require('express')
const multer = require('multer')
const moment = require('moment')
const uuidv4 = require('uuid/v4');
const bodyParser = require('body-parser')
const fs = require('fs')
const videoLib = require('node-video-lib')
const app = express()
const db = require('./db')
let Video = require('./models/videoModel')
//***** STAND LIBRARIES CONFIGURATION **********//
app.use(bodyParser.urlencoded({extended:true}))
app.use(bodyParser.json())
app.set('views', 'views')
app.set('view engine', 'ejs')
app.set(db)
//***** MULTER CONFIGURATION ***************//
let storage = multer.diskStorage({
destination: function(req, file, cb){
cb(null, './uploads')
},
filename: function(req, file, cb){
let mime = file.mimetype.split('/')[1]
let uuid = uuidv4()
cb(null, uuid + "." + mime)
}
})
function fileFilter(req, file, cb){
const extension = file.mimetype.split('/')[0];
if(extension !== 'video'){
return cb(new Error('Something went wrong. Wrong file format'), false);
}
cb(null, true);
};
var upload = multer({storage:storage, fileFilter: fileFilter})
const uploadHandler = upload.single('video')
function uploadVideo(req, res, next){
uploadHandler(req, res, next, function(err){
if(req.fileValidationError){
res.send('Error when upload')
}
console.log(req.file.filename)
next()
})
}
//******************************************//
function newVideo(req, res){
let videoParams = {title: req.body.title, path: req.file.filename}
Video.create(videoParams, function(err, result){
if(err){
console.log(err)
}
else{
console.log("Video salvo com sucesso")
console.log(result)
res.send(result )
}
})
}
app.get('/videos/:id', function(req, res){
let path = req.params.id
Video.find({path:path}, function(err, result){
if(err){
console.log(err);
}
else{
if (true) {
console.log("The url is:" + req.url);
const path = req.url.split('/')[2]
console.log("Path:" + path);
var file = `./uploads/${path}`
var range = req.headers.range;
fs.stat(file, function(err, stats) {
var total = stats.size;
if(range){
console.log('RANGE: ' + range);
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
var end = positions[1] ? parseInt(positions[1], 10) : total - 1;
var chunksize = (end - start) + 1;
console.log(req.url, start, end);
res.writeHead(206, {
"Content-Range": "bytes " + start + "-" + end + "/" + total,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4"
});
fs.createReadStream(file, { start: start, end: end }).pipe(res);
} else {
res.writeHead(200, { 'Content-Length': total, 'Content-Type': 'video/mp4' });
fs.createReadStream(file).pipe(res);
res.render('videos', {videoData:result})//Erro: can't set header after they're sent
}
});
} else {
console.log(req.url + ' (static)');
next();
}
}
})
})
app.get('/', function(req, res){
Video.find({}, function(err, result){
if(err){
console.log(err);
}
else{
console.log();
res.render('home', {videoData:result})
}
})
})
app.post('/upload', uploadVideo, newVideo)
app.listen(3000, ()=>{
console.log("Server running on port 3000")
})
You can't use both
res.writeHead();
and
res.render();
Read more: Error: Can't set headers after they are sent to the client
I currently have a node.js script sitting on Azure that gets a file (via a download URL link to that file) and base64 encodes it, and then sends this base64 encoded file back to the request source. The problem I am running into is performance based. The script below, in some instances, is timing out a separate application by having a run time over 30 seconds. The file in question on one of these timeouts was under a MB in size. Any ideas?
The script:
const express = require('express');
const bodyParser = require('body-parser');
const https = require('https');
const fs = require('fs');
const request = require('request');
const util = require('util');
const port = process.env.PORT || 3000;
var app = express();
app.use(bodyParser.json());
app.post('/base64file', (req, res) => {
var fileURL = req.body.fileURL;
var listenerToken = req.body.listenerToken;
var testingData = {
fileURL: fileURL,
listenerToken: listenerToken
};
/*
Make sure correct token is used to access endpoint..
*/
if( listenerToken !== <removedforprivacy> ) {
res.status(401);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'You are not authorized'}));
} else if ( !fileURL ){
res.status(400);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'The request could not be understood by the server due to malformed syntax.'}));
} else {
https.get(fileURL, function(response) {
var data = [];
response.on('data', function(chunk) {
data.push(chunk);
}).on('end', function() {
//build the base64 endoded file
var buffer = Buffer.concat(data).toString('base64');
//data to return
var returnData = {
base64File: buffer
};
if( buffer.length > 0 ) {
res.setHeader('Content-Type', 'application/json');
res.status(200);
res.send(JSON.stringify(returnData));
} else {
res.setHeader('Content-Type', 'application/json');
res.status(404);
res.send(JSON.stringify({ error: 'File URL not found.'}));
}
});
});
}
});
app.listen(port, () => {
console.log('Server is up and running ' + port);
});
One idea: you are missing error handling.
If you get an error on the https.get(), you will just never send a response and the original request will timeout.
If I call "http://localhost:3000/?fqn=mfgthun.ch&port=80" then i get back the following string:
{"source":"U259636","destination":"mfgthun.ch","ips":[]}
The resoult should be the following:
{"source":"U259636","destination":"mfgthun.ch","ips":[{"ip":"87.237.169.85","reachable":false}]}
The part in the function Write is missing... Do i need to add another callback here? Or is there any other solution for that?
Why is it not waiting until the first app.get block has finished?
const express = require('express')
const app = express()
const isReachable = require('is-reachable');
var dns = require('dns');
var os = require('os');
var url = require('url');
var jsonResponse;
function Write (ipPort, ip) {
this.ipPort = ipPort;
this.ip = ip;
this.callback = function(reachable) {
var temp = {ip: ip, reachable: reachable };
global.jsonResponse.ips.push(temp);
};
}
app.get("/", function(httpRequest, httpResponse, next){
try {
var url_parts = url.parse(httpRequest.url, true);
var query = url_parts.query;
global.jsonResponse = { source: os.hostname(), destination: query.fqn, ips: [] };
dns.resolve4(query.fqn, function (err, addresses) {
if (err) throw err;
for(var i = 0; i < addresses.length; i++) {
var ipPort = addresses[i] + ':' + query.port;
var write = new Write(ipPort, addresses[i]);
isReachable(ipPort).then(write.callback);
};
});
} catch(err) {
console.log(err);
};
next();
});
app.get("/", function(httpRequest, httpResponse){
httpResponse.write(JSON.stringify(global.jsonResponse));
httpResponse.end();
});
app.listen(3000)