I have a function which is getting data from url as stream and then I am using pipe to write this data in file using fs.createWriteStream. I want when file is created then it should return something. My current implementation is not waiting for file creation and returns immediately. Here is the snippet
const fs = require('fs');
const path = require('path');
const axios = require("axios");
const FileUploadPath = "./uploads/";
const downloadFile = async (url, filePath) => {
const writer = fs.createWriteStream(filePath)
const response = await axios({
url,
method: 'GET',
responseType: 'stream'
})
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on('finish', async function (err) {
console.log(err);
console.log("file downloaded")
resolve(filePath)
}
)
writer.on('error', reject(null))
});
}
const file_resolvers = {
Query: {
async file_download(parent, args, context, info) {
const fileUrl = // some file url
var filePath = // path where downloaded file will be saved
console.log("Downloading File " + fileUrl);
const result = await downloadFile(fileUrl, filePath);
console.log("Result received: " + result);
},
},
};
module.exports = {
file_resolvers,
}
I am getting "file downloaded" message after "Result received: null". Which means it does not wait for "finish" to get complete and returns with null immediately.
Related
I need to download a file from a private gitlab server and I need the method to be synchronous. This was by previous async code and it works fine because I was using promises. But I'm having trouble converting it to synchronous. The other posts i've seen on SO either ended up using async code or didn't have options for headers.
const https = require('https');
const fs = require('fs');
const gitlabUrl = 'https://gitlab.custom.private.com';
const gitlabAcessToken = 'xmyPrivateTokenx';
const gLfilePath = '/api/v4/projects/1234/repository/files/FolderOne%2Ftest.txt/raw?ref=main';
const gLfileName='test.txt';
function downloadFileFromGitlab(filePath, fileName) {
return new Promise((resolve, reject) => {
var options = {
path: filePath,
headers: {
'PRIVATE-TOKEN': gitlabAccessToken
}
};
var url = gitlabUrl
var file = fs.createWriteStream(fileName);
const request = https.get(url, options, (response) => {
response.pipe(file);
file.on('finish', () => {
file.close();
resolve();
});
file.on('error', (err) => {
file.close();
reject(err);
});
});
request.on('error', error => {
throw console.error(error);
});
});
}
downloadFileFromGitlab(gLfilePath,gLfileName);
I was able to figure it out using curl
function downloadFileFromGitlab(filePath, fileName) {
let curlCommand = "curl -s " + gitlabUrl + filePath + " -H 'PRIVATE-TOKEN:" + gitlabAccessToken +"'";
let file = child_process.execSync(curlCommand);
fse.writeFileSync(fileName,file);
}
I have built a AWS Lambda function with custom container image. I am trying to convert an excel file to pdf with Libreoffice - getting the file from S3 and saving it to a file and converting it to pdf and then uploading it back to S3.
Here the code.
const fs = require('fs');
const getStream = require('get-stream');
const { Readable } = require('stream')
const { S3Client, GetObjectCommand, PutObjectCommand } = require("#aws-sdk/client-s3");
const libre = require('libreoffice-convert');
const path = require('path');
exports.handler = async (event) => {
const bucket = event.queryStringParameters.bucket;
const file = event.queryStringParameters.file;
const convertedFile = event.queryStringParameters.convertedFile;
if (event.queryStringParameters['warmup'] !== undefined) {
return {
result: true,
message: 'warmed up'
}
}
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const response = await client.send(command);
const objectData = response.Body;
const writeStream = fs.createWriteStream("/tmp/sample.xlsx");
objectData.pipe(writeStream);
var end = new Promise((resolve, reject) => {
objectData.on('close', resolve(true));
objectData.on('end', resolve(true));
objectData.on('error', reject(false));
});
let completed = await end;
if (completed) {
const extend = '.pdf'
const outputPath = `/tmp/sample${extend}`;
const enterPath = '/tmp/sample.xlsx';
var readingFile = new Promise((resolve, reject) => {
fs.readFile(enterPath, (err, data)=>{
if (err) {
reject(false);
}
resolve(data);
});
});
var fileData = await readingFile;
var converting = new Promise((resolve, reject) => {
libre.convert(fileData, extend, undefined, (err, done) => {
if (err) {
reject(false)
}
fs.writeFileSync(outputPath, done);
resolve(true)
});
})
var converted = await converting;
if (converted) {
var convertedFileStream = fs.createReadStream(outputPath);
const uploadCommand = new PutObjectCommand({ Bucket: bucket, Key: convertedFile, Body: convertedFileStream });
const lastResponse = await client.send(uploadCommand);
const returnResponse = {
result: true,
message: 'success',
bucket: event.queryStringParameters.bucket,
file: event.queryStringParameters.file,
convertedFile: event.queryStringParameters.convertedFile
};
if (event.queryStringParameters['returnEvent'] !== undefined) {
returnResponse['returnEvent'] = event;
}
return returnResponse;
}
}
return completed;
};
However, I am getting this error at time. Sometimes, it is success, but, sometimes it throws this error.
{
"errorType": "Error",
"errorMessage": "false",
"stack": [
"Error: false",
" at _homogeneousError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:56:16)",
" at postError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:72:34)",
" at done (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:99:13)",
" at fail (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:113:13)",
" at /function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:148:24",
" at processTicksAndRejections (internal/process/task_queues.js:97:5)"
]
}
I dont know Nodejs on a great deal so I think if the code is not written the correct way. Any ideas what I am doing wrong here ?
Like #hoangdv when I logged errors I came to know that the file saving to the disk was not correct. So, I changed the area of the code where it saves to like this and then it worked.
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const { Body } = await client.send(command);
await new Promise((resolve, reject) => {
Body.pipe(fs.createWriteStream(filePath))
.on('error', err => reject(err))
.on('close', () => resolve())
})
const excelFile = fs.readFileSync(filePath);
I can't understand why this function exit before image is downloaded and saved.
I need to wait image will be saved before exiting.
function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = async (fileUrl, downloadFolder, filename) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
await response.data.pipe(fs.createWriteStream(localFilePath));
console.log("Successfully downloaded file!");
} catch (err) {
throw new Error(err);
}
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
downloadFile(IMAGE_URL, rel_path, filename);
}
The main problem is that using
axios({...})
you need to specify
url: imgUrl
More generally speaking:
avoid unnecessary nested functions
clean your unused and unnecessary params
Here a functioning example:
'use strict'
const fs = require('fs')
const path = require('path')
const axios = require('axios')
async function downloadImage(imgUrl, saveRelPath, saveName) {
const _path = path.resolve(__dirname, saveRelPath, saveName)
const writer = fs.createWriteStream(_path)
const response = await axios({
url: imgUrl,
method: 'GET',
responseType: 'stream',
})
response.data.pipe(writer)
return new Promise((resolve, reject) => {
writer.on('finish', resolve)
writer.on('error', reject)
})
}
async function anotherFunction() {
const IMG_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg'
const REL_PATH = '.'
const NAME = 'test.jpg'
console.log('Get and save image')
await downloadImage(IMG_URL, REL_PATH, NAME)
console.log('Image saving done, other stuff here')
}
anotherFunction()
EDIT: as response to
I can't understand why this function exit before image is downloaded and saved
the stream piping doesn't return a promise, so you can't await it.
You pipe a stream into a write stream that is an EventEmitter, so you should use a listener.
You can try that, it should work. for the pipeline() explaination, see this answer Promise completed before file is written. I promisified your flow to be able to resolve as soon as the stream process ends (note that i don t know about the other parts of your code, like if axios does not return a stream it will not work)
const { pipeline } = require('stream')
async function downloadImagefromRemote(url_immagine, filename) {
console.log('[2] Salvo l\'immagine remota in locale');
const downloadFile = (fileUrl, downloadFolder, filename) => {
return new Promise(async(resolve, reject) => {
// Get the file name
const fileName = path.basename(fileUrl);
// The path of the downloaded file on our machine
const localFilePath = path.resolve(__dirname, downloadFolder, filename);
try {
const response = await axios({ // <----- here jump out
method: "GET",
url: fileUrl,
responseType: "stream",
});
pipeline(
response.data,
fs.createWriteStream(localFilePath),
e => {
if(e) reject(e)
else resolve("Successfully downloaded file!")
}
)
} catch (err) {
reject(Error(err))
}
})
};
const IMAGE_URL = 'https://www.kindacode.com/wp-content/uploads/2021/01/test.jpg';
let rel_path = __dirname + '/../../public/images/';
try{
const confirmDownload = await downloadFile(IMAGE_URL, rel_path, filename);
console.log(confirmDownload)
} catch(e) {
console.log('catched err:', e)
}
}
I am aware that similar questions have been asked before. Still I am not able to solve the problem i have. I need to load a bunch of images before executing another part of code.
(async () => {
const urls = <array of urls>
await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
console.log('done');
})();
async function loadImageToPath(url, localPath) {
const file = fs.createWriteStream(localPath);
return await http.get(url, function (response) {
console.log('Image loaded: ' + localPath);
response.pipe(file);
});
}
Can someone please share some light on this!
thanks a lot
Map is returning an array of promises, to wait all promises to resolve use Promise.all(), MDN reference link
(async () => {
const urls = <array of urls>
const promises = await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
const responses = await Promise.all(promises) // this line waits all promises to resolve
console.log('done');
})();
I made some changes to the code and it is working now. I thought that http is returning a promise by itself. With a wrapper that returns a promise it is working now.
(async () => {
const urls = <array of urls>
await urls.map(url => {
const filename = path.basename(url);
const localPath = imagesPath + '/' + filename;
return loadImageToPath(url, localPath);
});
console.log('done');
})();
async function loadImageToPath(url, localPath) {
const file = fs.createWriteStream(localPath);
return new Promise((resolve, reject) => {
http.get(url, function (response) {
console.log('Image loaded: ' + localPath);
response.pipe(file);
resolve();
});
});
}
I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});