I need to download a file from a private gitlab server and I need the method to be synchronous. This was by previous async code and it works fine because I was using promises. But I'm having trouble converting it to synchronous. The other posts i've seen on SO either ended up using async code or didn't have options for headers.
const https = require('https');
const fs = require('fs');
const gitlabUrl = 'https://gitlab.custom.private.com';
const gitlabAcessToken = 'xmyPrivateTokenx';
const gLfilePath = '/api/v4/projects/1234/repository/files/FolderOne%2Ftest.txt/raw?ref=main';
const gLfileName='test.txt';
function downloadFileFromGitlab(filePath, fileName) {
return new Promise((resolve, reject) => {
var options = {
path: filePath,
headers: {
'PRIVATE-TOKEN': gitlabAccessToken
}
};
var url = gitlabUrl
var file = fs.createWriteStream(fileName);
const request = https.get(url, options, (response) => {
response.pipe(file);
file.on('finish', () => {
file.close();
resolve();
});
file.on('error', (err) => {
file.close();
reject(err);
});
});
request.on('error', error => {
throw console.error(error);
});
});
}
downloadFileFromGitlab(gLfilePath,gLfileName);
I was able to figure it out using curl
function downloadFileFromGitlab(filePath, fileName) {
let curlCommand = "curl -s " + gitlabUrl + filePath + " -H 'PRIVATE-TOKEN:" + gitlabAccessToken +"'";
let file = child_process.execSync(curlCommand);
fse.writeFileSync(fileName,file);
}
Related
I can't understand why this function exit before image is downloaded and saved.
I need to wait image will be saved before exiting.
function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = async (fileUrl, downloadFolder, filename) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
await response.data.pipe(fs.createWriteStream(localFilePath));
console.log("Successfully downloaded file!");
} catch (err) {
throw new Error(err);
}
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
downloadFile(IMAGE_URL, rel_path, filename);
}
The main problem is that using
axios({...})
you need to specify
url: imgUrl
More generally speaking:
avoid unnecessary nested functions
clean your unused and unnecessary params
Here a functioning example:
'use strict'
const fs = require('fs')
const path = require('path')
const axios = require('axios')
async function downloadImage(imgUrl, saveRelPath, saveName) {
const _path = path.resolve(__dirname, saveRelPath, saveName)
const writer = fs.createWriteStream(_path)
const response = await axios({
url: imgUrl,
method: 'GET',
responseType: 'stream',
})
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on('finish', resolve)
writer.on('error', reject)
})
}
async function anotherFunction() {
const IMG_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg'
const REL_PATH = '.'
const NAME = 'test.jpg'
console.log('Get and save image')
await downloadImage(IMG_URL, REL_PATH, NAME)
console.log('Image saving done, other stuff here')
}
anotherFunction()
EDIT: as response to
I can't understand why this function exit before image is downloaded and saved
the stream piping doesn't return a promise, so you can't await it.
You pipe a stream into a write stream that is an EventEmitter, so you should use a listener.
You can try that, it should work. for the pipeline() explaination, see this answer Promise completed before file is written. I promisified your flow to be able to resolve as soon as the stream process ends (note that i don t know about the other parts of your code, like if axios does not return a stream it will not work)
const { pipeline } = require('stream')
async function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = (fileUrl, downloadFolder, filename) => {
return new Promise(async(resolve, reject) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
pipeline(
response.data,
fs.createWriteStream(localFilePath),
e => {
if(e) reject(e)
else resolve("Successfully downloaded file!")
}
)
} catch (err) {
reject(Error(err))
}
})
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
try{
const confirmDownload = await downloadFile(IMAGE_URL, rel_path, filename);
console.log(confirmDownload)
} catch(e) {
console.log('catched err:', e)
}
}
I have a function which is getting data from url as stream and then I am using pipe to write this data in file using fs.createWriteStream. I want when file is created then it should return something. My current implementation is not waiting for file creation and returns immediately. Here is the snippet
const fs = require('fs');
const path = require('path');
const axios = require("axios");
const FileUploadPath = "./uploads/";
const downloadFile = async (url, filePath) => {
const writer = fs.createWriteStream(filePath)
const response = await axios({
url,
method: 'GET',
responseType: 'stream'
})
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on('finish', async function (err) {
console.log(err);
console.log("file downloaded")
resolve(filePath)
}
)
writer.on('error', reject(null))
});
}
const file_resolvers = {
Query: {
async file_download(parent, args, context, info) {
const fileUrl = // some file url
var filePath = // path where downloaded file will be saved
console.log("Downloading File " + fileUrl);
const result = await downloadFile(fileUrl, filePath);
console.log("Result received: " + result);
},
},
};
module.exports = {
file_resolvers,
}
I am getting "file downloaded" message after "Result received: null". Which means it does not wait for "finish" to get complete and returns with null immediately.
I am aware that similar questions have been asked before. Still I am not able to solve the problem i have. I need to load a bunch of images before executing another part of code.
(async () => {
const urls = <array of urls>
await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
console.log('done');
})();
async function loadImageToPath(url, localPath) {
const file = fs.createWriteStream(localPath);
return await http.get(url, function (response) {
console.log('Image loaded: ' + localPath);
response.pipe(file);
});
}
Can someone please share some light on this!
thanks a lot
Map is returning an array of promises, to wait all promises to resolve use Promise.all(), MDN reference link
(async () => {
const urls = <array of urls>
const promises = await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
const responses = await Promise.all(promises) // this line waits all promises to resolve
console.log('done');
})();
I made some changes to the code and it is working now. I thought that http is returning a promise by itself. With a wrapper that returns a promise it is working now.
(async () => {
const urls = <array of urls>
await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
console.log('done');
})();
async function loadImageToPath(url, localPath) {
const file = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
http.get(url, function (response) {
console.log('Image loaded: ' + localPath);
response.pipe(file);
resolve();
});
});
}
I want to download a series of ts files, but every time I run this code, it will be stuck in the download process of a file without any error message.
However, if you put the url in the program in the address bar of the browser, or use a tool like wget, you can still download it.
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
const puppeteer = require('puppeteer');
var https = require('https');
var fs = require('fs');
async function doRequest(url) {
return new Promise(function (resolve) {
var req = https.get(url, function (response) {
resolve(response);
});
req.end('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
req.on('error', function(e) {
sleep(3000).then(() => {
console.log('re try to get ts ...'+e);
doRequest(url); })
});
});
}
(async() => {
for(let i = 21; i<100; i++)
{
var ts_url = 'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000'+i+'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MmFmN2JkNzEtODc3Mi00OThmLThjNzQtN2Y5NTQ3ZTgxYjA2IiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzEzNzU4fX19XX0_&Signature=hFOZlf5Iqf7k-LqVms1FT4LKoEM8psMVkCsdjoHGhKztNTzROyGgfHg34RYr3ezffQaQV6drXBuID1NMypQhwSXgJ-ZRAoGC3KnKtrfm9bSpK2Wq97sZf97D5PbBn7wNaJhmfvbztym-cRknztepOqM2v~KvCz6~esS99TOsWbtQFBZWLPacp5MV3v5eZ4wh2WJXX1jDqI1XmpZ0jyU2oJCXOgVbvU1aF86E7duvniDrbhmS1R00~tWTAETBbmBSubDw-7fGq7XzeZcFRfXbdwb0a9KwsAGh54lj1UBUMsDzEtH8vI8r9aC~MnFIRub1KxsFSOzUlRLYRp4GsPJQiQ__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=2af7bd71-8772-498f-8c74-7f9547e81b06';
await console.log('downloading '+i);
var file_ts_path = fs.createWriteStream(i+'.ts');
let responseRes = await doRequest(ts_url);
await responseRes.pipe(file_ts_path);
}
})();
You need to download a file, write it to stream, and then pipe it.
https://nodejs.org/api/https.html#https_https_get_url_options_callback
https://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options
var https = require('https');
var fs = require('fs');
function doRequest(url, dest) {
return new Promise(function (resolve) {
https.get(url, res => {
let data = '';
res.on('data', (buffer) => {
data += buffer; // save downloaded data
});
res.on('end', () => {
var stream = fs.createWriteStream(dest);
stream.write(data, 'binary');
stream.on('finish', () => {
resolve('File Saved !');
});
res.pipe(stream);
});
}).on('error', function (e) {
reject('re try to get ts ...' + e);
});
});
}
for (let i = 21; i < 100; i++) {
var ts_url =
'https://hls2.videos.sproutvideo.com/1e7c8a8ff0518509452c7eb2e75a2a1f/e84883ff69cb66752bd6783cdbaa35fb/video/720_000' +
i +
'.ts?Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9obHMyLnZpZGVvcy5zcHJvdXR2aWRlby5jb20vMWU3YzhhOGZmMDUxODUwOTQ1MmM3ZWIyZTc1YTJhMWYvZTg0ODgzZmY2OWNiNjY3NTJiZDY3ODNjZGJhYTM1ZmIvKi50cz9zZXNzaW9uSUQ9MDZkN2ZhOWMtMDgxOS00Y2YwLTk0M2QtNzA2MGQzOGY2N2RkIiwiQ29uZGl0aW9uIjp7IkRhdGVMZXNzVGhhbiI6eyJBV1M6RXBvY2hUaW1lIjoxNTQ2NzY3NjY4fX19XX0_&Signature=DTCQWVIdInCe2YIf-fxD4RDEHOXGUDK2pUwxV0cKPi0m~WwlLYIEFSmQkAbK-oV-uLU93E1O2TGizvrMDp6voFVnm-jLaOur1JRlJDBCP7T8KEYrkkU3Y3grZAKHmi0gQiVpVIKRgo7gnDKwMZ1NjosQPbaf1XDMpuHxAyKfPGgIRLpSEp4BZ1dqcfzs-YyYQzNaK-a3tYONmpyID3bZnF8sn2pMZonArCz24BQL0wEfXeS3HqxwVv85z641kKxQBGd~8lG88qUTpJCvqWmIZhikzWjGQPY~6ezgJMKhjJQIoPMVGZehT~NcAzPwXo84kd5ksaOdbh4paHsUe1096A__&Key-Pair-Id=APKAIB5DGCGAQJ4GGIUQ&sessionID=06d7fa9c-0819-4cf0-943d-7060d38f67dd';
doRequest(ts_url, i + '.ts')
.then(console.log)
.catch(console.log);
}
I decided to post this after extensive searching here (1, 2, 3 ) and here (1, 2) and many, many other related posts. I am loosing hope, but will not give up that easily :)
I'm using multer to upload a PNG image to mongo database:
const storage = new GridFsStorage({
url: 'mongodb://my_database:thisIsfake#hostName/my_database',
file: (req, file) => {
return new Promise((resolve, reject) => {
crypto.randomBytes(16, (err, buf) => { // generating unique names to avoid duplicates
if (err) {
return reject(err);
}
const filename = buf.toString('hex') + path.extname(file.originalname);
const fileInfo = {
filename: filename,
bucketName: 'media',
metadata : {
clientId : req.body.client_id // added metadata to have a reference to the client to whom the image belongs
}
};
resolve(fileInfo);
});
});
}
});
const upload = multer({storage}).single('image');
Then I create a stream and pipe it to response:
loader: function (req, res) {
var conn = mongoose.createConnection('mongodb://my_database:thisIsfake#hostName/my_database');
conn.once('open', function () {
var gfs = Grid(conn.db, mongoose.mongo);
gfs.collection('media');
gfs.files.find({ metadata : {clientId : req.body.id}}).toArray(
(err, files) => {
if (err) throw err;
if (files) {
const readStream = gfs.createReadStream(files[0].filename); //testing only with the first file in the array
console.log(readStream);
res.set('Content-Type', files[0].contentType)
readStream.pipe(res);
}
});
});
}
Postman POST request to end point results in response body being displayed as an image file:
In the front end I pass the response in a File object, read it and save the result in a src attribute of img:
findAfile(){
let Data = {
id: this.$store.state.StorePatient._id,
};
console.log(this.$store.state.StorePatient._id);
visitAxios.post('http://localhost:3000/client/visits/findfile', Data )
.then(res => {
const reader = new FileReader();
let file = new File([res.data],"image.png", {type: "image/png"});
console.log('this is file: ',file);
reader.readAsDataURL(file); // encode a string
reader.onload = function() {
const img = new Image();
img.src = reader.result;
document.getElementById('imgContainer').appendChild(img);
};
})
.catch( err => console.error(err));
}
My File object is similar to the one I get when using input field only bigger:
This is original file:
When inspecting element I see this:
Looks like data URI is where it should be, but it's different from the original image on file input:
Again, when I want to display it through input element:
onFileSelected(event){
this.file = event.target.files[0];
this.fileName = event.target.files[0].name;
const reader = new FileReader();
console.log(this.file);
reader.onload = function() {
const img = new Image();
img.src = reader.result;
document.getElementById('imageContainer').appendChild(img);
};
reader.readAsDataURL(this.file);
}
I get this:
But when reading it from the response, it is corrupted:
Postman gets it right, so there must be something wrong with my front-end code, right? How do I pass this gfs stream to my html?
I managed to make a POST request to fetch an image from MongoDB and save it in the server dir:
const readStream = gfs.createReadStream(files[0].filename);
const wstream = fs.createWriteStream(path.join(__dirname,"uploads", "fileToGet.jpg"));
readStream.pipe(wstream);
Then, I just made a simple GET request by adding an absolute path to the and finally delete the file after successful response:
app.get('/image', function (req, res) {
var file = path.join(dir, 'fileToGet.jpg');
if (file.indexOf(dir + path.sep) !== 0) {
return res.status(403).end('Forbidden');
}
var type = mime[path.extname(file).slice(1)] || 'text/plain';
var s = fs.createReadStream(file);
s.on('open', function () {
res.set('Content-Type', type);
s.pipe(res);
});
s.on('end', function () {
fs.unlink(file, ()=>{
console.log("file deleted");
})
});
s.on('error', function () {
res.set('Content-Type', 'text/plain');
res.status(404).end('Not found');
});