NodeJS server loading Loop - node.js

const http = require('http');
const fs = require('fs');
const requests = require('requests');
const homefile = fs.readFileSync("./home.html", 'utf-8');
const replaceVal = (tempVal, orgVal) => {
let temperature = tempVal.replace(`{%tempVal%}`, orgVal.main.temp);
return temperature;
}
const server = http.createServer((req, res) => {
if (req.url == '/') {
requests('https://api.openweathermap.org/data/2.5/weather?q=Pune&appid=4bebabe44d8d1f07b79b9bbc8ed9dd33')
.on('data', function (chunk) {
const objdata = JSON.parse(chunk);
const arrayDatda = [objdata];
console.log(arrayDatda);
const realtimeData = arrayDatda.map((val) => replaceVal(homefile, val)).join("")
// console.log(realtimeData)
res.write(realtimeData)
})
.on('end', function (err) {
if (err) {
console.log('connection closed due to errors', err);
}
res.end("working");
});
}
})
server.listen(80, '127.0.0.1');
Whenever I try to restart the server it just goes on loading loop and didn't show any results until I stop the loading

I tried running your code without the file replacement logic you've written and got this error.
_http_outgoing.js:679
throw new ERR_INVALID_ARG_TYPE('first argument',
^
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be of type string or an instance of Buffer. Received an instance of Array
at write_ (_http_outgoing.js:679:11)
at ServerResponse.write (_http_outgoing.js:644:15)
You have to send a supported content-type to res.write.
const realtimeData = arrayDatda.map((val) => replaceVal(homefile, val)).join("")
realtimeData is an Array and is not supported.
const http = require('http');
const fs = require('fs');
const requests = require('requests');
// const homefile = fs.readFileSync("./home.html", 'utf-8');
// const replaceVal = (tempVal, orgVal) => {
// let temperature = tempVal.replace(`{%tempVal%}`, orgVal.main.temp);
// return temperature;
// }
const server = http.createServer((req, res) => {
if (req.url == '/') {
requests('https://api.openweathermap.org/data/2.5/weather?q=Pune&appid=4bebabe44d8d1f07b79b9bbc8ed9dd33')
.on('data', function (chunk) {
const objdata = JSON.parse(chunk);
const arrayDatda = [objdata];
console.log(arrayDatda);
// const realtimeData = arrayDatda.map((val) => replaceVal(homefile, val)).join("")
// console.log(realtimeData)
// converting arrayDatda to string and sending works
res.write(JSON.stringify(arrayDatda))
})
.on('end', function (err) {
if (err) {
console.log('connection closed due to errors', err);
}
res.end("working");
});
}
})
server.listen(8080, '127.0.0.1');
So after replacing your file with data that you've received from API, you have to serve that file in res.write, not the realtimeDatda variable which is an Array.
I suggest going through the basics of node http server again and learn about how to serve different content types and static files.
Here is a good beginner blog:
https://www.digitalocean.com/community/tutorials/how-to-create-a-web-server-in-node-js-with-the-http-module

Related

data fatching form api to html in node

I was learn create wather app by using node.js form youtube and i just copy the code that he was writing but there code is work properly and my code dose not work
in my code I don't understand this line and after type this line my code is give arror before this every thing is right
const rtd = adata.map((val) => replaceVal(index, val)).join(" ");
res.write(rtd);
the code in this image is written by me and this code is note work
const fs = require('fs');
const http = require('http');
var requests = require('requests');
const replaceVal = (tempval, orgVal) => {
let temperature = tempval.replace('{% tempval %}', orgVal.current.temp_c);
temperature = temperature.replace('{% location %}', orgVal.location.region);
temperature = temperature.replace('{% country %}', orgVal.location.country);
return temperature;
}
const index = fs.readFileSync('index.html', 'utf-8');
const server = http.createServer((req, res) => {
if (req.url == '/') {
requests('https://api.weatherapi.com/v1/current.json?key=fbffa2e936464bd292b70117221308&q=india&aqi=no')
.on('data', (chunk) => {
const jdata = JSON.parse(chunk);
const adata = [jdata];
// console.log(adata)
// console.log(adata[0].current.temp_c)
const rtd = adata.map((val) => replaceVal(index, val)).join(" ");
res.write(rtd);
})
.on('end', function (err) {
if (err) return console.log('connection closed due to errors', err);
res.end();
// console.log('')
});
}
})
server.listen(8000, "127.0.0.1");
this is screen shot of the video

How to store the readStream files of Node.js into Redis and also how to retrieve the stored readStream files from Redis?

I tried converting the readStream (Image) to string and then storing it in Redis. Then retrieving the string from Redis and converting it back to readStream. But it didn't work out.
function getFile(fileKey) {
console.log(fileKey);
const downloadParams = {
Key: fileKey,
Bucket: bucketName,
};
return s3.getObject(downloadParams).createReadStream();
}
exports.getFile = getFile;
For converting stream to string I'm using stream-to-string. It gets converted and stored in Redis.
const { getFile } = require("../s3");
const redis = require("redis");
const client = redis.createClient();
var toString = require("stream-to-string");
exports.getFileFromS3Controller = async (req, res) => {
console.log(req.params);
const path = req.params.path;
const key = req.params.key;
const readStream = getFile(path + "/" + key);
toString(readStream).then(function (msg) {
// Set data to Redis
client.setex(key, 3600, msg);
});
readStream.pipe(res);
};
On Retrieving from the Redis I am not getting it.
const redis = require("redis");
const client = redis.createClient(null, null, { detect_buffers: true });
const Readable = require("stream").Readable;
// Cache middleware
function cache(req, res, next) {
const { path, key } = req.params;
client.get(key, (err, data) => {
if (err) throw err;
if (data !== null) {
var s = new Readable();
s.push(data);
s.push(null);
s.pipe(res);
} else {
next();
}
});
}
router.get("/:path/:key", cache, getFileFromS3Controller);
You are not calling next. Another mistake is that the stream is not being saved anywhere in the req so you can access later from the controller. From what I see you are writing it directly in res which is a problem because after this you cannot use res anymore to send anything else.
Here it is the code (not tested)
exports.getFileFromS3Controller = (req, res) => {
if (req.fileStream) {
req.fileStream.pipe(res);
return
}
console.log(req.params);
const path = req.params.path;
const key = req.params.key;
const readStream = getFile(path + "/" + key);
toString(readStream).then(function (msg) {
// Set data to Redis
client.setex(key, 3600, msg);
// Conver string to readable
const readable = new Readable();
readable.push(msg);
readable.push(null);
readable.pipe(res);
});
};
function cache(req, res, next) {
const { path, key } = req.params;
client.get(key, (err, data) => {
if (err) throw err;
if (data !== null) {
var s = new Readable();
s.push(data);
s.push(null);
req.fileStream = s;
}
next();
});
}
Edit I fixed a mistake in my answer because a Readable stream cannot be rewinded.

Heroku Nodejs Webhosting

Im trying to host a election website.
My Current Method explained:
There is server.js file with serves the public file. If its a POST request(sent by frontend.js to send data to server) edits the poll.json file based on the data sent.
I have a public file having index.html, frontend.js, poll.json where poll.json stores all the data.
My current code work all well in my localhost.
But when running it in Heroku I get a error in the POST request line saying 'POST http://localhost:53390/ net::ERR_CONNECTION_REFUSED1
*********.herokuapp.com/:1 Uncaught (in promise) TypeError: Failed to fetch'
My server.js code:
const createServer = require('http').createServer;
const express = require('express');
const app = express();
let pollData = require(__dirname+'/public/poll.json');
const fs = require('fs');
var all_usn_no=[];
const listening_port=process.env.PORT || 4000;
function get_roll_and_usn(pollData){
for(var i=0;i<=pollData.students.length-1;i++){
//all_roll_no.push(pollData.students[i][0]);
all_usn_no.push(pollData.students[i][1]);
}
}
function roll_to_row(in_usn){
get_roll_and_usn(pollData)
return all_usn_no.indexOf(in_usn);
}
function write_vote(votes){
var checking_row=roll_to_row(votes[1]);
pollData.students[checking_row]=votes;
fs.writeFile(__dirname+'/public/poll.json', JSON.stringify(pollData), (err) => {
if (err) throw err;
console.log('Data written to file');
});
}
write_vote([listening_port,0]);
app.use(express.static(__dirname+'/public'));
app.get('/', (req, res) => {
res.sendFile(__dirname);// + '/index.html');
});
app.post('/', (req, res) => {
let body = '';
req.on('data', data => body += data)
req.on('end', () => {
res.writeHead(200, {
'Content-Type': 'application/json',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'POST, GET',
});
body=body.split(",");
console.log(body.toString());
write_vote(body);
res.end(`{ "response": "${body ? body : 'No body sent' }" }`);
})
});
app.listen(listening_port, () => {
console.log('Example app listening at http://localhost:'+listening_port)
});
my frontend.js program:
var roll='';
var client_ip='';
var usn='';
var date='';
const all_roll_no=[]
const all_usn_no=[]
var port='';
function check(roll,usn){
var data='';
const url='poll.json';
const Http = new XMLHttpRequest();
Http.open("GET", url);
Http.send();
Http.onload = () =>{
data=JSON.parse(Http.response);
get_roll_and_usn(data);
check_validity(roll,usn,data);
}
}
function get_roll_and_usn(pollData){
for(var i=0;i<=pollData.students.length-1;i++){
//all_roll_no.push(pollData.students[i][0]);
all_usn_no.push(pollData.students[i][1]);
}
}
function usn_to_row(in_usn){
return all_usn_no.indexOf(in_usn);
}
function check_validity(checking_roll,checking_usn,data){
var checking_row=usn_to_row(checking_usn);
port=data.students[0][0];
//if(all_roll_no.indexOf(checking_roll)>=0 && checking_usn==all_usn_no[all_roll_no.indexOf(checking_roll)] && data.students[checking_row][2]==0){
if(all_usn_no.indexOf(checking_usn)>=0 && data.students[checking_row][2]==0){
//console.log("valid");
document.getElementById("page_2").style.display = "none";
document.getElementById("page_3").style.display = "block";
fetch('https://api.ipify.org/?format=json').then(results=> results.json()).then(data => client_ip=data.ip);
}
else{
alert("You cannot vote/ You have already voted")
//console.log('invalid');
}
}
document.getElementById("startbutton").onclick = function(){
roll = document.getElementById("roll_no").value;
usn = document.getElementById("usn_no").value;
date = Date();
check(roll,usn);
}
document.getElementById("next").onclick = function() {
document.getElementById("page_1").style.display = "none";
document.getElementById("page_2").style.display = "block";
}
document.getElementById("finish").onclick = function() {
var splb=[document.getElementById("splb1").checked,document.getElementById("splb2").checked,document.getElementById("splb3").checked,document.getElementById("splb4").checked,document.getElementById("splb5").checked,document.getElementById("splb6").checked,document.getElementById("splb7").checked];
var splg=[document.getElementById("splg1").checked,document.getElementById("splg2").checked,document.getElementById("splg3").checked,document.getElementById("splg4").checked,document.getElementById("splg5").checked,document.getElementById("splg6").checked,document.getElementById("splg7").checked];
var asplb=[document.getElementById("asplb1").checked,document.getElementById("asplb2").checked,document.getElementById("asplb3").checked,document.getElementById("asplb4").checked,document.getElementById("asplb5").checked,document.getElementById("asplb6").checked,document.getElementById("asplb7").checked];
var asplg=[document.getElementById("asplg1").checked,document.getElementById("asplg2").checked,document.getElementById("asplg3").checked,document.getElementById("asplg4").checked,document.getElementById("asplg5").checked,document.getElementById("asplg6").checked,document.getElementById("asplg7").checked];
var csb=[document.getElementById("csb1").checked,document.getElementById("csb2").checked,document.getElementById("csb3").checked,document.getElementById("csb4").checked,document.getElementById("csb5").checked,document.getElementById("csb6").checked,document.getElementById("csb7").checked];
var csg=[document.getElementById("csg1").checked,document.getElementById("csg2").checked,document.getElementById("csg3").checked,document.getElementById("csg4").checked,document.getElementById("csg5").checked,document.getElementById("csg6").checked,document.getElementById("csg7").checked];
var acsb=[document.getElementById("acsb1").checked,document.getElementById("acsb2").checked,document.getElementById("acsb3").checked,document.getElementById("acsb4").checked,document.getElementById("acsb5").checked,document.getElementById("acsb6").checked,document.getElementById("acsb7").checked];
var acsg=[document.getElementById("acsg1").checked,document.getElementById("acsg2").checked,document.getElementById("acsg3").checked,document.getElementById("acsg4").checked,document.getElementById("acsg5").checked,document.getElementById("acsg6").checked,document.getElementById("acsg7").checked];
var scb=[document.getElementById("scb1").checked,document.getElementById("scb2").checked,document.getElementById("scb3").checked,document.getElementById("scb4").checked,document.getElementById("scb5").checked,document.getElementById("scb6").checked,document.getElementById("scb7").checked];
var scg=[document.getElementById("scg1").checked,document.getElementById("scg2").checked,document.getElementById("scg3").checked,document.getElementById("scg4").checked,document.getElementById("scg5").checked,document.getElementById("scg6").checked,document.getElementById("scg7").checked];
var ascb=[document.getElementById("ascb1").checked,document.getElementById("ascb2").checked,document.getElementById("ascb3").checked,document.getElementById("ascb4").checked,document.getElementById("ascb5").checked,document.getElementById("ascb6").checked,document.getElementById("ascb7").checked];
var ascg=[document.getElementById("ascg1").checked,document.getElementById("ascg2").checked,document.getElementById("ascg3").checked,document.getElementById("ascg4").checked,document.getElementById("ascg5").checked,document.getElementById("ascg6").checked,document.getElementById("ascg7").checked];
var vote=[String(splb.indexOf(true)),String(splg.indexOf(true)),String(asplb.indexOf(true)),String(asplg.indexOf(true)),String(csb.indexOf(true)),String(csg.indexOf(true)),String(acsb.indexOf(true)),String(acsg.indexOf(true)),String(scb.indexOf(true)),String(scg.indexOf(true)),String(ascb.indexOf(true)),String(ascg.indexOf(true))]
var update=[String(roll),String(usn),"1",String(client_ip),String(date)].concat(vote);
if (update.indexOf("-1")<0){
alert("Pls vote for all posts")
}
else{
document.getElementById("page_1").style.display = "none";
document.getElementById("page_2").style.display = "none";
document.getElementById("page_3").style.display = "none";
fetch('http://localhost:'+port,{method:'POST',body:update}).then(results=> results.json()).then(console.log);
//console.log(update);
alert("Your vote has been registered")
}
}
You can just ignore the function as they are just to process the data and do necessary funtions.
Main problem: PORT request from frontend.js to server.js sending data to edit the poll.json file return error.
Thanks in advance.
Try using const PORT = process.env.PORT || 3333;
with
app.listen(PORT, () => {
console.log(`Server is starting on ${PORT}`);
})

How to concat chunks of incoming binary into video (webm) file node js?

I am trying to upload chunks of base64 to node js server and save those chunks into one file
let chunks = [];
app.post('/api', (req, res) => {
let {blob} = req.body;
//converting chunks of base64 to buffer
chunks.push(Buffer.from(blob, 'base64'));
res.json({gotit:true})
});
app.post('/finish', (req, res) => {
let buf = Buffer.concat(chunks);
fs.writeFile('finalvideo.webm', buf, (err) => {
console.log('Ahh....', err)
});
console.log('SAVED')
res.json({save:true})
});
Problem with the above code is video is not playable I don't why Am I really doing something wrong and I've also tried writable streams it is not working either
UPDATE - I
Instead of sending blobs I've implemented to send binary but even though I am facing a problem like TypeError: First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.
client.js
postBlob = async blob => {
let arrayBuffer = await new Response(blob).arrayBuffer();
let binary = new Uint8Array(arrayBuffer)
console.log(binary) // logging typed Uint8Array
axios.post('/api',{binary})
.then(res => {
console.log(res)
})
};
server.js
let chunks = [];
app.post('/api', (req, res) => {
let {binary} = req.body;
let chunkBuff = Buffer.from(binary) // This code throwing Error
chunks.push(chunkBuff);
console.log(chunkBuff)
res.json({gotit:true})
});
//Somehow combine those chunks into one file
app.post('/finish', (req, res) => {
console.log('Combinig the files',chunks.length);
let buf = Buffer.concat(chunks);
console.log(buf) //empty buff
fs.writeFile('save.webm', buf, (err) => {
console.log('Ahh....', err)
});
res.json({save:true})
});
UPDATE - II
I am able to receive the binary chunk and append to a stream but in the final video only first chunk is playing I don't know what happened to other chunks and the video ends.
code
const writeMyStream = fs.createWriteStream(__dirname+'/APPENDED.webm', {flags:'a', encoding:null});
app.post('/api', (req, res) => {
let {binary} = req.body;
let chunkBuff = Buffer.from(new Uint8Array(binary));
writeMyStream.write(chunkBuff);
res.json({gotit:true})
});
UPDATE - III
my client code | Note: I've tried other ways to upload blobs I've commented out
customRecordStream = stream => {
let recorder = new MediaStreamRecorder(stream);
recorder.mimeType = 'video/webm;codecs=vp9';
recorder.ondataavailable = this.postBlob
recorder.start(INT_REC)
};
postBlob = async blob => {
let arrayBuffer = await new Response(blob).arrayBuffer();
let binary = new Uint8Array(arrayBuffer)
axios.post('/api',{binary})
.then(res => {
console.log(res)
})
// let binaryUi8 = new Uint8Array(arrayBuffer);
// let binArr = Array.from(binaryUi8);
// // console.log(new Uint8Array(arrayBuffer))
//
// console.log(blob);
// console.log(binArr)
// let formData = new FormData();
// formData.append('fname', 'test.webm')
// formData.append("file", blob);
//
// console.log(formData,'Checjk Me',blob)
// axios({
// method:'post',
// url:'/api',
// data:formData,
// config: { headers: {'Content-Type': 'multipart/form-data' }}
// }).then(res => {
// console.log(res,'FROM SERBER')
//
// })
//
//
// .then(res => {
// console.log(res)
// })
// this.blobToDataURL(blob, (blobURL) => {
//
// axios.post('/api',{blob:blobURL})
// .then(res => {
// console.log(res)
// })
// })
};
I was able to get this working by converting to base64 encoding on the front-end with the FileReader api. On the backend, create a new Buffer from the data chunk sent and write it to a file stream. Some key things with my code sample:
I'm using fetch because I didn't want to pull in axios.
When using fetch, you have to make sure you use bodyParser on the backend
I'm not sure how much data you're collecting in your chunks (i.e. the duration value passed to the start method on the MediaRecorder object), but you'll want to make sure your backend can handle the size of the data chunk coming in. I set mine really high to 50MB, but this may not be necessary.
I never close the write stream explicitly... you could potentially do this in your /final route. Otherwise, createWriteStream defaults to AutoClose, so the node process will do it automatically.
Full working example below:
Front End:
const mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', handleSourceOpen, false);
let mediaRecorder;
let sourceBuffer;
function customRecordStream(stream) {
// should actually check to see if the given mimeType is supported on the browser here.
let options = { mimeType: 'video/webm;codecs=vp9' };
recorder = new MediaRecorder(window.stream, options);
recorder.ondataavailable = postBlob
recorder.start(INT_REC)
};
function postBlob(event){
if (event.data && event.data.size > 0) {
sendBlobAsBase64(event.data);
}
}
function handleSourceOpen(event) {
sourceBuffer = mediaSource.addSourceBuffer('video/webm; codecs="vp8"');
}
function sendBlobAsBase64(blob) {
const reader = new FileReader();
reader.addEventListener('load', () => {
const dataUrl = reader.result;
const base64EncodedData = dataUrl.split(',')[1];
console.log(base64EncodedData)
sendDataToBackend(base64EncodedData);
});
reader.readAsDataURL(blob);
};
function sendDataToBackend(base64EncodedData) {
const body = JSON.stringify({
data: base64EncodedData
});
fetch('/api', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body
}).then(res => {
return res.json()
}).then(json => console.log(json));
};
Back End:
const fs = require('fs');
const path = require('path');
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
const server = require('http').createServer(app);
app.use(bodyParser.urlencoded({ extended: true }));
app.use(bodyParser.json({ limit: "50MB", type:'application/json'}));
app.post('/api', (req, res) => {
try {
const { data } = req.body;
const dataBuffer = new Buffer(data, 'base64');
const fileStream = fs.createWriteStream('finalvideo.webm', {flags: 'a'});
fileStream.write(dataBuffer);
console.log(dataBuffer);
return res.json({gotit: true});
} catch (error) {
console.log(error);
return res.json({gotit: false});
}
});
Inspired by #willascend answer:
Backend-side:
app.use(express.raw());
app.post('/video-chunck', (req, res) => {
fs.createWriteStream('myvideo.webm', { flags: 'a' }).write(req.body);
res.sendStatus(200);
});
Frontend-side:
mediaRecorder.ondataavailable = event => {
if (event.data && event.data.size > 0) {
fetch(this.serverUrl + '/video-chunck', {
method: 'POST',
headers: {'Content-Type': 'application/octet-stream'},
body: event.data
});
}
};
My express version is 4.17.1
i faced the same problem today
as a solution in back-end i used fs.appendfile
fs.appendFile(Path, rawData, function (err) {
if (err) throw err;
console.log('Chunck Saved!');
})

nodejs express server stops unexpectedly

I have a server built with express and socket IO. The problem is that every time the server just stops responding 2 minutes after first client connection. Things I have already done/checked:
Express logs show no errors.
socket.IO logs show no errors on server side.
socket.IO logs show ping timeout on client side (after a few successful pings).
Listening to the uncaughtException/UnhandeledRejection doesn't help.
So, when I start the server, and GET /login through the browser, without doing anything else, the server will stop responding on its own after ~2 minutes, and the only way to close it is closing the cmd (CTRL+C doesn't work even after listening to SIGINT).
I'll post the relevant parts of the code here:
server.js
const http = require('http');
const express = require('express');
const socketIO = require('socket.io');
let app = express();
let server = http.createServer(app);
var io = socketIO(server);
app.get(`/login`, (req, res) => {
console.log(`got GET /login`);
res.sendFile(publicPath + '/login.html');
});
io.on(`connection`, (socket) => {
socket.on(`login`, async (details, cb) => {
login(details.username, details.password, async (res) => {
if (res === true) {
let user = new User(details.username, details.password);
let token = await user.generateAuthToken();
cb(`/set?token=${token}`);
} else if (res === false) {
socket.emit(`loginFailed`, 'Password incorrect. Please try again!')
} else if (res === `Username doesn't exist`) {
socket.emit(`loginFailed`, res)
}
});
});
socket.on('getAllLeads', async (callback) => {
leads = await getAllLeads();
callback(leads);
});
socket.on('getSomeLeads', async (options, callback) => {
leads = await getSomeLeads(options);
callback(leads);
});
socket.on(`newFile`, ({text, words, options, fileName}) => {
console.log('Starting...');
words.forEach((wordSet, index) => {
wordSet.forEach(word => {
regexp = `\\s+${word}+\\s`;
re = new RegExp(regexp);
text = text.replace(re, `${options[index]}`);
});
});
fs.writeFileSync(`${fileName}.txt`, text, 'utf8');
console.log(`Finished writing to ${fileName}.txt successfully!`);
});
});
server.listen(3000, () => {
console.log(chalk.yellow(`Server is up on port ${port}`));
});
login.js
const socket = io();
socket.on(`reconnect_error`, (err) => {
console.log(err);
});
socket.on(`connect`, () => {
console.log('connected to login page');
if (getQueryVariable('failed')) {
$('.error, .error p').css('opacity', 1);
$('.error p').text('Unauthorized! PLEASE log in.');
}
socket.on(`loginFailed`, (reason) => {
$('.error, .error p').css('opacity', 1);
$('.error p').text(reason);
console.log(`got login failed`);
});
$('#form').submit((e) => {
e.preventDefault();
let username = $('#username').val();
let password = $('#pass').val();
socket.emit(`login`, {username, password}, (path) => {
window.location.href = path;
});
});
});
function getQueryVariable(variable)
{
var query = window.location.search.substring(1);
var vars = query.split("&");
for (var i=0;i<vars.length;i++) {
var pair = vars[i].split("=");
if (pair[0] == variable) return pair[1];
}
return (false);
}

Resources