URL inside ffmpeg(fs.createStreamReader(url)) [duplicate] - node.js

on nodejs documentation, the streams section says I can do fs.createReadStream(url || path).
But, when I actually do that It tells me Error: ENOENT: no such file or directory.
I just want to pipe the video from a readable to a writable stream, But I'm stuck on creating a readable one.
my code:
const express = require('express')
const fs = require('fs')
const url = 'https://www.example.com/path/to/mp4Video.mp4'
const port = 3000
app.get('/video', (req, res) => {
const readable = fs.createReadStream(url)
})
app.listen(port, () => {
console.log('listening on port ' + port)
})
the ERROR:
listening on port 3000
events.js:291
throw er; // Unhandled 'error' event
^
Error: ENOENT: no such file or directory, open 'https://www.example.com/path/to/mp4Video.mp4'
Emitted 'error' event on ReadStream instance at:
at internal/fs/streams.js:136:12
at FSReqCallback.oncomplete (fs.js:156:23) {
errno: -2,
code: 'ENOENT',
syscall: 'open',
path: 'https://www.example.com/path/to/mp4Video.mp4'
}
PS: https://www.example.com/path/to/mp4Video.mp4 IS NOT THE ACTUAL URL

fs.createReadStream() does not work with http URLs only file:// URLs or filename paths. Unfortunately, this is not described in the fs doc, but if you look at the source code for fs.createReadStream() and follow what it calls you can find that it ends up calling fileURULtoPath(url) which will throw if it's not a file: URL.
function fileURLToPath(path) {
if (typeof path === 'string')
path = new URL(path);
else if (!isURLInstance(path))
throw new ERR_INVALID_ARG_TYPE('path', ['string', 'URL'], path);
if (path.protocol !== 'file:')
throw new ERR_INVALID_URL_SCHEME('file');
return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path);
}
It would suggest using the got() library to get yourself a readstream from a URL:
const got = require('got');
const mp4Url = 'https://www.example.com/path/to/mp4Video.mp4';
app.get('/video', (req, res) => {
got.stream(mp4Url).pipe(res);
});
More examples described in this article: How to stream file downloads in Nodejs with Got.
You can also use the plain http/https modules to get the readstream, but I find got() to be generally useful at a higher level for lots of http request things so that's what I use. But, here's code with the https module.
const https = require('https');
const mp4Url = 'https://www.example.com/path/to/mp4Video.mp4';
app.get("/", (req, res) => {
https.get(mp4Url, (stream) => {
stream.pipe(res);
});
});
More advanced error handling could be added to both cases.

Related

how to create a readable stream from a remote url in nodejs?

on nodejs documentation, the streams section says I can do fs.createReadStream(url || path).
But, when I actually do that It tells me Error: ENOENT: no such file or directory.
I just want to pipe the video from a readable to a writable stream, But I'm stuck on creating a readable one.
my code:
const express = require('express')
const fs = require('fs')
const url = 'https://www.example.com/path/to/mp4Video.mp4'
const port = 3000
app.get('/video', (req, res) => {
const readable = fs.createReadStream(url)
})
app.listen(port, () => {
console.log('listening on port ' + port)
})
the ERROR:
listening on port 3000
events.js:291
throw er; // Unhandled 'error' event
^
Error: ENOENT: no such file or directory, open 'https://www.example.com/path/to/mp4Video.mp4'
Emitted 'error' event on ReadStream instance at:
at internal/fs/streams.js:136:12
at FSReqCallback.oncomplete (fs.js:156:23) {
errno: -2,
code: 'ENOENT',
syscall: 'open',
path: 'https://www.example.com/path/to/mp4Video.mp4'
}
PS: https://www.example.com/path/to/mp4Video.mp4 IS NOT THE ACTUAL URL
fs.createReadStream() does not work with http URLs only file:// URLs or filename paths. Unfortunately, this is not described in the fs doc, but if you look at the source code for fs.createReadStream() and follow what it calls you can find that it ends up calling fileURULtoPath(url) which will throw if it's not a file: URL.
function fileURLToPath(path) {
if (typeof path === 'string')
path = new URL(path);
else if (!isURLInstance(path))
throw new ERR_INVALID_ARG_TYPE('path', ['string', 'URL'], path);
if (path.protocol !== 'file:')
throw new ERR_INVALID_URL_SCHEME('file');
return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path);
}
It would suggest using the got() library to get yourself a readstream from a URL:
const got = require('got');
const mp4Url = 'https://www.example.com/path/to/mp4Video.mp4';
app.get('/video', (req, res) => {
got.stream(mp4Url).pipe(res);
});
More examples described in this article: How to stream file downloads in Nodejs with Got.
You can also use the plain http/https modules to get the readstream, but I find got() to be generally useful at a higher level for lots of http request things so that's what I use. But, here's code with the https module.
const https = require('https');
const mp4Url = 'https://www.example.com/path/to/mp4Video.mp4';
app.get("/", (req, res) => {
https.get(mp4Url, (stream) => {
stream.pipe(res);
});
});
More advanced error handling could be added to both cases.

EPIPE Error when uploading large file using nodejs

I was following this article to setup a nodejs server on my local machine (which has 16 gb memory and about 170gb free disk-space) and uploaded a 20 gb file, for the first couple of times the file got uploaded successfully, but after a while i started getting EPIPE error:
error FetchError: request to http://localhost:3200/upload failed, reason: write EPIPE
at ClientRequest.<anonymous> (/Volumes/FreeAgent GoFlex Drive/Test/multer-project/node_modules/node-fetch/lib/index.js:1455:11)
at ClientRequest.emit (events.js:327:22)
at Socket.socketErrorListener (_http_client.js:467:9)
at Socket.emit (events.js:315:20)
at emitErrorNT (internal/streams/destroy.js:100:8)
at emitErrorCloseNT (internal/streams/destroy.js:68:3)
at processTicksAndRejections (internal/process/task_queues.js:84:21) {
type: 'system',
errno: 'EPIPE',
code: 'EPIPE'
}
When i checked, the file got uploaded partially and was about 28mb in size. I tried uploading the file from both Postman, browser and a nodejs script, but got the same EPIPE error message. I am not sure why is this happening, googling the error message didn't help. I am not sure how to overcome this. Following is my server and client code.
// server.js
const express = require("express"); // Express Web Server
const busboy = require("connect-busboy"); // Middleware to handle the file upload https://github.com/mscdex/connect-busboy
const path = require("path"); // Used for manipulation with path
const fs = require("fs-extra");
const app = express(); // Initialize the express web server
app.use(
busboy({
highWaterMark: 2 * 1024 * 1024 // Set 2MiB buffer
})
); // Insert the busboy middle-ware
const uploadPath = path.join(__dirname, "uploads/"); // Register the upload path
fs.ensureDir(uploadPath); // Make sure that he upload path exits
/**
* Create route /upload which handles the post request
*/
app.route("/upload").post((req, res, next) => {
req.pipe(req.busboy); // Pipe it trough busboy
req.busboy.on("file", (fieldname, file, filename) => {
console.log(`Upload of '${filename}' started`);
// Create a write stream of the new file
const fstream = fs.createWriteStream(path.join(uploadPath, filename));
// Pipe it trough
file.pipe(fstream);
// On finish of the upload
fstream.on("close", () => {
console.log(`Upload of '${filename}' finished`);
res.send("ok");
});
});
});
/**
* Serve the basic index.html with upload form
*/
app.route("/").get((req, res) => {
res.writeHead(200, { "Content-Type": "text/html" });
res.write(
'<form action="upload" method="post" enctype="multipart/form-data">'
);
res.write('<input type="file" name="fileToUpload"><br>');
res.write('<input type="submit">');
res.write("</form>");
return res.end();
});
const server = app.listen(3200, function() {
console.log(`Listening on port ${server.address().port}`);
});
and my client code is:
// client.js
const fs = require("fs");
const FormData = require("form-data");
const fetch = require("node-fetch");
var formdata = new FormData();
formdata.append(
"file",
fs.createReadStream("/Users/phantom007/My Documents/35gb.myfile")
);
var requestOptions = {
method: "POST",
body: formdata,
redirect: "follow"
};
fetch("http://localhost:3200/upload", requestOptions)
.then(response => response.text())
.then(result => console.log(result))
.catch(error => console.log("error", error));
Answering my own question.
After strugging for a long time i figured out that this error was coming because the number of bytes getting written on the same is larger than the number of bytes sent to the server, so in my client code, i changed
this
fs.createReadStream("/Users/phantom007/My Documents/35gb.myfile")
to this
fs.createReadStream("/Users/phantom007/My Documents/35gb.myfile", { highWaterMark: 2 * 1024 * 1024 })

got error when try to fetch some data from open weather api

This app is to get small piece of data from open weather map through api, app.js:
const express = require("express");
const https = require('https');
const app = express();
app.get("/", function (req, res) {
const url = "https://api.openweathermap.org/data/2.5/weather?q=Khartoum&appid=d244334364579178494bb3c4528e218b&units=metric"
https.get(url, function (response) {
response.on("data", function (data) {
const weatherData = JSON.parse(data);
res.write(weatherData.weather[0].description);
res.write(weatherData.weather[0].id);
res.send();
});
});
});
app.listen("3000", function () {
console.log("runnign on port 3000.");
});
the error that triggered in terminal when I reloaded the page:
_http_outgoing.js:671
throw new ERR_INVALID_ARG_TYPE('first argument',
^
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be of type string or an instance of Buffer. Received type number (804)
at write_ (_http_outgoing.js:671:11)
at ServerResponse.write (_http_outgoing.js:636:15)
at IncomingMessage.<anonymous> (/home/foola/Desktop/appBrewery/WeatherProject/app.js:18:17)
at IncomingMessage.emit (events.js:315:20)
at IncomingMessage.Readable.read (_stream_readable.js:512:10)
at flow (_stream_readable.js:985:34)
at resume_ (_stream_readable.js:966:3)
at processTicksAndRejections (internal/process/task_queues.js:84:21) {
code: 'ERR_INVALID_ARG_TYPE'
So what's the wrong here exactly?
mybe weatherData.weather[0].id is a number type likely integer?
try
res.write(weatherData.weather[0].id+'');
As the error message indicated, res.write accepts buffer or string only, weatherData.weather[0].id is of type int/number, therefore you might want to stringify it first before passing it to res.write.
res.write(weatherData.weather[0].id.toString())

Upload file to ftp with node.js and npm ftp (ftp from GoDaddy)

I'm triying to upload files to my FTP which lives on GoDaddy Hosting Plan Deluxe with Express.js. After a lot of search found the NPM ftp library
I've followed the instructions to connect to my ftp account (a special account to upload files) but now i can't upload the file. Next, my code so far:
const express = require('express');
var Client = require('ftp');
const app = express();
c.connect({
host: thehost,
port: 21,
user: theuser,
password: thepass,
});
var c = new Client();
c.on('ready', function() {
c.list(function(err, list) {
if (err) throw err;
console.dir(list);
c.end();
});
});
app.post('/files', async (req, res) => {
let myFile = req.files.filesBox;
let cleanName = myFile.name.replace(/[`~!##$%^&*()|+\=?;:'"áéíóúâêîûãõçÇñÑ, <>\{\}\[\]\\\/]/gi, '-');
var c = new Client();
c.on('ready', function() {
c.put(cleanName, function(err) {
if (err) throw err;
c.end();
});
});
c.connect();
});
const port = process.env.PORT || 8080;
app.listen(port, () =>
console.log(`App on port: ${port}`)
);
As you can see, i get the file name using let myFile = req.files.filesBox; (filesBox is the name of the form), then i remove all the special characters with let cleanName and then, send the file. My problem is that the response messages is:
Error: connect ECONNREFUSED 127.0.0.1:21
at Object._errnoException (util.js:992:11)
at _exceptionWithHostPort (util.js:1014:20)
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1186:14)
Someone knows why is not working? I´ve readed on a forum that the response comes 'cause i'm not connecting properly to the FTP. But when the conection begins, there's no error response, and the dir list is printed on the console like this:
[ { name: 'images',
type: 'd',
size: 0,
date: 2019-10-13T20:28:00.000Z },
{ name: 'remote-folder',
type: 'd',
size: 0,
date: 2019-10-06T05:13:00.000Z } ]
I'm missing something? Someone had the same error?
I made a test with FileZilla and the testing images uploaded without problems.
Hope you can help me.
As i said, i'm working with Node, Express and NPM FTP library

Cannot find file in Node even though it has correct path

After executing this code:
const filename = "../../.dburl"
const url = fs.readFileSync(filename, 'utf-8')
I recieve the following error:
Error: ENOENT: no such file or directory, open '../../.dburl'
What I know so far:
1) The filepath is correct.
2) My application has permission to read the file.
3) .dburl is not read even when stored in the same directory as the application.
Any help is much appreciated ... Thanks!
You can use the module-level variable __dirname to get the directory that contains the current script. Then you can use path.resolve() to use relative paths.
console.log('Path of file in parent dir:', require('path').resolve(__dirname, '../app.js'));
READING FILE ON SAME LEVEL
//the server object listens on port 8080
const PORT = 8080;
var http = require("http");
var fs = require("fs");
var path = require("path");
//create a server object:
http
.createServer((req, res) => {
console.log("READING FILE: ", path.resolve(__dirname, "input.txt"));
fs.readFile(path.resolve(__dirname, "input.txt"), (err, data) => {
//error handling
if (err) return console.error(err);
//return file content
console.log("FILE CONTENT: " + data.toString());
res.write(data.toString());
res.end();
console.log("READ COMPLETED");
});
})
.listen(PORT);
READING FILE ON OTHER LEVEL:
//the server object listens on port 8080
const PORT = 8080;
var http = require("http");
var fs = require("fs");
var path = require("path");
//create a server object:
http
.createServer((req, res) => {
console.log("READING FILE: ", path.resolve(__dirname, "./mock/input.txt"));
fs.readFile(path.resolve(__dirname, "./mock/input.txt"), (err, data) => {
//error handling
if (err) return console.error(err);
//return file content
console.log("FILE CONTENT: " + data.toString());
res.write(data.toString());
res.end();
console.log("READ COMPLETED");
});
})
.listen(PORT);
I'm guessing you're confusing the current working directory of your script for the location of your script. It's hard to say without knowing the structure of your project and where you're calling the script from.
Assuming your shell's working directory is /, .dburl is at /.dburl and your script is located at /foo/bar/script.js. If you run node foo/bar/script, you could read .dburl with readFileSync('./dburl'). However if you run cd foo/bar; node ./script, you would need to read .dburl with readFileSync('../../.dburl').
This is because the working directory of your script is equal to the working directory of the shell you launched it in.

Resources