Piping Firebase stream to Express response result in HPE_INVALID_CONSTANT - node.js

I am trying to stream a video (mp4) from firebase storage to <video> on client. What I'm doing is using createReadStream and piping it to Express response object. However on the client this error is thrown:
Proxy error: Could not proxy request /movie/600d31f192e0941f9c4b4773/stream from localhost:3000 to http://localhost:5000/.
[1] See https://nodejs.org/api/errors.html#errors_common_system_errors for more information (HPE_INVALID_CONSTANT).
const streamMovie = async (req, res) => {
const range = req.headers.range
const movie = await Movie.findById(req.params.id)
const bucket = firebase.storage().bucket()
// Get video size
const videoFile = bucket.file(movie.videoFileUrl)
const [metadata] = await videoFile.getMetadata()
const videoSize = metadata.size
// Parse range
const parts = range.replace('bytes=', '').split('-')
const start = parseInt(parts[0], 10)
const end = parts[1] ? parseInt(parts[1], 10) : videoSize - 1
res.writeHead(206, {
'Content-Type': 'video/mp4',
'Content-Range': `bytes ${start}-${end}/${videoSize}`,
'Content-Length': `${end - start + 1}`,
'Accept-Ranges': 'bytes',
})
videoFile.createReadStream({ start, end }).pipe(res)
}
I could not pinpoint the source of the error since I'm pretty inexperienced with streams. Any help is appreciated. Thanks!

If you want to stream a video file to the express response object, you can use the code example from the client library official docs. I have tested the following sample successfully:
const express = require('express')
const app = express()
const port = 3000
app.get('/', (req, res) => {
const {Storage} = require('#google-cloud/storage');
const storage = new Storage();
const bucket = storage.bucket('bucket123');
const remoteFile = bucket.file('videotest123.mp4');
res.writeHead(200, {
'Content-Type': 'video/mp4',
'Content-Range': 'bytes=0-',
'Accept-Ranges': 'bytes',
})
remoteFile.createReadStream()
.on('error', function(err) {
res.send('there was an error');
})
.on('response', function(response) {})
.on('end', function() {})
.pipe(res);
})
app.listen(port, () => {
console.log(`App listening at http://localhost:${port}`)
})
If you want to request only certain parts of the video (aka Google Cloud Storage object), you can specify within the createReadStream method the start and end options.
const remoteFile = bucket.file('videotest123');
logFile.createReadStream({
start: 10000,
end: 20000
})
.on('error', function(err) {})
.pipe(res);
In regards to the HPE_INVALID_CONSTANT message, I have found here that the message indicates an web server response malformed. In this case the resobject from the code.
I hope you find this useful

Related

req.body is empty when sending data in server, in local works

I'm trying to send data to index.js, in localhost it's works correctly but when I deploy to my server it's not working. I see problem seems similar :
here. but it's not work for me
client.js :
sending data to index.js /subscribeA
await fetch("https://example.com:30000/subscribeA", {
method: "post",
body: JSON.stringify({ dataSubscribe: subscription, dataNotif: dataNotif}),
headers: {
"Content-Type": "application/json"
},
mode : 'no-cors'
});
console.log("Push Sent...");
}
then in index.js :
var express = require('express');
const port = process.env.PORT || 30000;
const app = express();
const http = require('http');
const https = require('https');
const fs = require('fs');
var server = http.createServer(app);
const privateKey = fs.readFileSync('/etc/letsencrypt/live/example.com/privkey.pem', 'utf8');
const certificate = fs.readFileSync('/etc/letsencrypt/live/example.com/cert.pem', 'utf8');
const credintials = { key: privateKey, cert: certificate };
server = https.createServer(credintials, app);
server.listen(port, () => console.log(`Server started on port ${port}`));
const io = require("socket.io")(server, {
// allowEIO3 = true,
cors: {
credentials:true,
origin: '*',
methods: ["GET", "POST"]
}
})
const webpush = require("web-push");
const bodyParser = require("body-parser");
const path = require("path");
// Set static path
app.use(express.static(path.join(__dirname)));
app.use(bodyParser.json());
const publicVapidKey =
"BJthRQ5maDga7OSXsPC1ftGw-n16F7zQBEN7EUD6XxcfTTvrLGWSIG7y_JxiWtVlCFua0S8MTB5rPziBqNx1qIo";
const privateVapidKey = "3KsvdasAdSoCdsd0dIG_o9B0Ozvl1XDwI63JRKNIWBM";
webpush.setVapidDetails(
"mailto:test#test.com",
publicVapidKey,
privateVapidKey
);
// Subscribe Route
app.post("/subscribeA", (req, res) => {
console.log(req.body);
// Get pushSubscription object
// console.log(req.body.dataSubscribe);
const subscription = req.body.dataSubscribe;
const dataNotif = req.body.dataNotif;
if(dataNotif == null){
console.log('kosong rek');
}else{
console.log(dataNotif);
}
// Send 201 - resource created
res.status(201).json({});
// Create payload
const payload = JSON.stringify({ head: "yobro", content: "kontennnya bro"});
// Pass object into sendNotification
webpush
.sendNotification(subscription, payload)
.catch(err => console.error(err));
});
io.on('connection', function (socket) {
socket.on('notifikasi', function (data) {
io.sockets.emit('notifikasi', {
isi_notif: data.isi_notif});
});
});
Request Payload
{dataSubscribe: {,…}, dataNotif: {head: "#username", content: "yo"}}
dataNotif: {head: "#username", content: "yo"}
dataSubscribe: {,…}
So, it's sending data correctly, but in index.js when I use console.log(req.body) it return empty array {}.
Using no-cors in your fetch() means that the content-type can only be one of the following: "application/x-www-form-urlencoded", "multipart/form-data", or "text/plain" (some more explanation here). So, your "application/json" is not allowed and thus the server doesn't properly read/parse the body.
Only simple headers are allowed with no-cors which you can read about here.
You will have to stop using no-cors with your fetch to use JSON or change the content to use one of the allowed content-types.
FYI, in cases like this, it's generally helpful to log out the important expected aspects of the incoming request. In this case, you could log the content-type and see what you're getting. You could also look at the request as it's being sent from the browser in the network tab of the Chrome debugger. Either would probably show you something related to the content-type.

Trying to apply a watermark in real time with Node.js + FFMPEG

I'm trying to write a streaming video server using Node.js express. the main task of the video server is to apply a watermark on the video. Here is my code
const express = require("express");
const fs = require("fs");
const path = require("path");
const ffmpeg = require("fluent-ffmpeg");
const ffmpegInstaller = require("#ffmpeg-installer/ffmpeg");
ffmpeg.setFfmpegPath(ffmpegInstaller.path);
const app = express();
app.get("/", function (req, res) {
const path = req.query.video;
const WATERMARK_PATH = req.query.id + `.png`;
const stat = fs.statSync(path);
const fileSize = stat.size;
const range = req.headers.range;
if (range) {
const parts = range.replace(/bytes=/, "").split("-");
const start = parseInt(parts[0], 10);
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1;
const chunksize = end - start + 1;
const head = {
"Content-Range": `bytes ${start}-${end}/${fileSize}`,
"Accept-Ranges": "bytes",
"Content-Length": chunksize,
"Content-Type": "video/mp4",
};
res.writeHead(206, head);
new ffmpeg(fs.createReadStream(path, { start, end }))
.input(WATERMARK_PATH)
.complexFilter(
"overlay='x=if(eq(mod(n\\,18)\\,0)\\,sin(random(1))*w\\,x):y=if(eq(mod(n\\,18)\\,0)\\,sin(random(1))*h\\,y)'",
)
.outputOptions("-movflags frag_keyframe+empty_moov")
.toFormat("mp4")
.pipe(res, { end: true });
} else {
const head = {
"Content-Length": fileSize,
"Content-Type": "video/mp4",
};
res.writeHead(200, head);
new ffmpeg(fs.createReadStream(path))
.input(WATERMARK_PATH)
.complexFilter(
"overlay='x=if(eq(mod(n\\,18)\\,0)\\,sin(random(1))*w\\,x):y=if(eq(mod(n\\,18)\\,0)\\,sin(random(1))*h\\,y)'",
)
.outputOptions("-movflags frag_keyframe+empty_moov")
.toFormat("mp4")
.pipe(res, { end: true });
// fs.createReadStream(path).pipe(res)
}
});
app.listen(3020, function () {
console.log("App is running on port 3020");
});
As a result, the video doesn't play, and the following error appears in the console.
Error:
c:\myapp>node server.js
App is running on port 3000
events.js:292
throw er; // Unhandled 'error' event
^
Error: Output stream closed
at Timeout._onTimeout (c:\myapp\node_modules\fluent-ffmpeg\lib\processor.js:491:25)
at listOnTimeout (internal/timers.js:549:17)
at processTimers (internal/timers.js:492:7)
Emitted 'error' event on FfmpegCommand instance at:
at emitEnd (c:\myapp\node_modules\fluent-ffmpeg\lib\processor.js:424:16)
at Timeout._onTimeout (c:\myapp\node_modules\fluent-ffmpeg\lib\processor.js:491:17)
at listOnTimeout (internal/timers.js:549:17)
at processTimers (internal/timers.js:492:7)
If you remove the assignment of headers and contact the server, we will download the video, that is, FFMPEG works, a watermark is added.
Q: How do I set up video playback?

sending and recieving a file in Nodejs with Express

I'm writing a server and a client with Node.js, the server uses express and the client uses axios.
I'm trying to send an image file from the client to the server. I found somewhere here this bit of code for the client:
let file = fs.createReadStream(file_path);
let form_data = new FormData();
form_data.append("picture", file);
let post_config = {
method: "post",
url: SERVER_PICTURE_URL,
headers: {"Content-Type": "multipart/form-data"},
data: form_data
}
axios(post_config).then(_ => {console.log("sent");} );
But I can't figure out what's supposed to be on the server side. I've tried the most obvious solution, writing response.data or response.form to a file, but both are undefined.
Is there some parser I'm supposed to use? And if so, how?
I'm not sure about the client that you wrote, but in express, you need to use express-fileupload package for getting the picture from req.files
const express = require('express');
const fileupload = require("express-fileupload");
const app = express();
const port = 3000;
app.use(fileupload());
app.post('/picture', (req, res) => {
const files=req.files;
res.send(files)
})
app.listen(port, () => {
console.log(`app listening at http://localhost:${port}`)
})

nodejs http. large file transfer fails with 'ERR_STREAM_PREMATURE_CLOSE'

Im trying to send 10 GB data from a node client to a node http server. It works fine until i reach about 5 GB. Protocol is http.
It works fine until i reach about 5 GB. Error I get on server side is [ERR_STREAM_PREMATURE_CLOSE] and on client side RequestError: read ECONNRESET.
example server:
const http = require('http')
const fs = require('fs')
const util = require('util')
const stream = require('stream')
const pipeline = util.promisify(stream.pipeline)
const host = 'localhost'
const port = 4000
const server = http.createServer(async (req, res) => {
try {
const writeStream = fs.createWriteStream('file', { encoding: 'binary' })
await pipeline(req, writeStream)
} catch (e) {
console.log(e)
}
})
server.listen(port, host, () => {
console.log(`Server is running on http://${host}:${port}`)
})
example client
const got = require('got')
async function upload () {
await pipeline(fs.createReadStream(filename), got.stream.post('http://localhost:4000/upload'))
}
I have tried with different servers (raw http and express) and clients (raw node, request, node fetch). Ive also tried multipart with busboy. Same problem.
Im trying this running node v14.4.0 on a Mac.
I tried transferring a 10 gb file on my computer and it worked for me.
Server example:
const http = require('http');
const fs = require('fs');
const PORT = 8333;
const server = http.createServer((req, res) => {
const writeStream = fs.createWriteStream('./result.txt');
req.pipe(writeStream);
req.on('end', () => {
console.log('The file was successfully written.');
res.end('OK');
});
});
server.listen(PORT, () => {
console.log(`Server has been started on port ${server.address().port}`);
});
Client example
const http = require('http');
const fs = require('fs');
function upload() {
const readStream = fs.createReadStream('./test.txt');
const request = http.request('http://localhost:8333', { method: 'POST' }, (res) => console.log(`STATUS: ${res.statusCode}`));
readStream.pipe(request);
readStream.on('end', () => request.end());
}
upload();
I am not saying that this code is very correct, but you can try it, since it worked for me.
I ran this example on Node.js v12.4.1 and Windows 10.

Is there a way to stream download a mp3 file that is being converted on a nodejs server?

I am looking for a way to send the url to the nodejs server and respond the user with the mp3 file download.
I searched some examples, and read about requests and responses, but I am not sure what the problem really is.
This is the Javascript for the HTML:
var downloadBtn = document.querySelector('.download_button');
var URLinput = document.querySelector('#myUrl');
downloadBtn.addEventListener('click', () => {
console.log(`URL: ${URLinput.value}`);
sendURL(URLinput.value);
});
function sendURL(URL) {
window.location.href = `http://localhost:4000/download?URL=${URL}`;
}
This is the Javascript for the Nodejs server:
const express = require('express');
const cors = require('cors');
const ytdl = require('ytdl-core');
const app = express();
const ffmpeg = require('fluent-ffmpeg')
app.use(cors());
app.listen(4000, () => {
console.log('Server Works !!! At port 4000');
});
app.get('/download', (req,res) => {
var URL = req.query.URL;
res.header('Content-Disposition', 'attachment; filename="file.mp3"');
let stream = ytdl(URL, {
quality: 'highestaudio',
}); //HERE THE STREAM FILE IS SELECTED TO BE CONVERTED TO MP3
ffmpeg(stream)
.audioBitrate(128)
.pipe(res); // HERE IS CONVERTED AND WHERE I WANT IT TO SEND IT AS A DOWNLOAD TO THE USER.
});
I expected it to stream download the file but instead it gets me to the nodejs server page to /download/url_to_vid

Resources