I am trying to upload files through a lambda function and request/response is working fine. Problem is each uploaded file is missing some data so the uploaded file is corrupted. e.g. if I try to upload a 5 Kb file, only 4.5 Kb is getting uploaded. This is confirmed with size variable from logs.
parseMultipart = async (event) => {
return new Promise((resolve, reject) => {
const parsedForm = {};
const bb = new busboy({
headers: {
'content-type': event.headers['Content-Type'] || event.headers['content-type']
}
});
bb.on('file', function (fieldname, file, filename, encoding, mimetype) {
var bufs = [];
var size = 0;
file
.on('data', async (data) => {
//bufs[bufs.length] = data;
await bufs.push(data);
size += data.length;
console.log('size:' + size);
})
.on('end', async () => {
console.log('size in end:' + size);
parsedForm[fieldname] = {
data: Buffer.concat(bufs),
filename: filename,
encoding: encoding,
mimetype: mimetype
};
});
})
.on('field', (fieldname, val) => {
parsedForm[fieldname] = val
})
.on('finish', async () => {
console.log("in finish:");
await resolve(parsedForm);
})
.on('close', () => {
console.log("in close");
resolve(parsedForm);
})
.on('error', error => reject(error))
bb.write(event.body, event.isBase64Encoded ? 'base64' : 'binary');
bb.end();
})
}
What is it that I am missing or doing differently? I have already checked relevant questions on SO for busboy.
Related
I read Pipe a stream to s3.upload()
but im having difficulty with I am not sure if that actually solves and I have tried.
What I am doing is a get call to www.example.com. this returns a stream, I want to upload that stream to s3.
heres my try.
fetch('https://www.example.com',fileName{
method: 'GET',
headers: {
'Authorization': "Bearer " + myAccessToken,
},
})
.then(function(response) {
return response.text();
})
.then(function(data) {
uploadToS3(data)
});
const uploadToS3 = (data) => {
// Setting up S3 upload parameters
const params = {
Bucket:myBucket,
Key: "fileName",
Body: data
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
output: ///File uploaded successfully. https://exampleBucket.s3.amazonaws.com/fileName.pdf
however this is blank.
I figured it out, but i did not keep using fetch.
and I actually download the file, then upload it. then delete the file.
function getNewFilesFromExampleDotCom(myAccessToken, fileName, fileKey) {
let url2 = 'https://example.com' + fileKey;
axios
.get(url2, {
headers: { 'Authorization': "Bearer " + myAccessToken },
responseType: 'stream',
})
.then(response => {
let file = fileName;
response.data.pipe(fs.createWriteStream(file))
let myFileInfo = [];
if( myFileInfo.length > 0){
myFileInfo.splice(0, myFileInfo.length)
}
myFileInfo.push(file)
processArray(myFileInfo)
console.log(file + " saved")
})
.catch(error => console.log(error));
}
async function processArray(array) {
for (const item of array) {
await delayedLog(item);
}
console.log('Downloaded!');
console.log('Uploading to s3!');
}
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function delayedLog(item) {
await delay();
uploadFiles(item)
}
async function uploadFiles(file){
uploadToS3List(file)
await new Promise((resolve, reject) => setTimeout(resolve, 1000));
deleteMyFiles(file)
}
const uploadToS3List = (fileName) => {
// Read content from the file
const fileContent = fs.readFileSync(fileName);
// Setting up S3 upload parameters
const params = {
Bucket:"myBucketName",
Key: fileName,
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
function deleteMyFiles(path){
fs.unlink(path, (err) => {
console.log(path + " has been deleted")
if (err) {
console.error(err)
return
}
})
}
Im trying to make a quick node script to download MP3s from a RSS feed. At the moment I have this :
const https = require('https');
const xml2js = require('xml2js');
const parser = new xml2js.Parser();
const fs = require('fs');
const URL_TO_PARSE = 'https://some-rss.feed.xml';
const req = https.get(URL_TO_PARSE, async (res) => {
let xml = '';
res.on('data', (stream) => {
xml = xml + stream;
});
res.on('end', () => {
parser.parseString(xml, async (err, result) => {
if (err) {
console.log(err);
} else {
let items = result.rss.channel[0].item;
await Promise.all(items.map(async (item) => {
let title = item.title[0];
let enclosure = item.enclosure[0];
let url = enclosure.$.url;
let filepath = `./${title}`;
console.log(`Downloading ${title} to ${filepath}`);
await download_audio_file(url, filepath);
}));
}
});
});
});
const download_audio_file = async (url, filepath) => {
https.get(url, (res) => {
const writeStream = fs.createWriteStream(filepath);
res.pipe(writeStream);
writeStream.on('finish', () => {
writeStream.close();
console.log('File downloaded');
Promise.resolve();
});
writeStream.on('error', (err) => {
console.log(err);
Promise.reject(err);
});
})
But it currently tried to download each one at the same time. Is there a better way to write this to download just one at a time - and possibly also track the % progress?
I see 2 problems with your code.
The first one is that download_audio_file is not returning a promise that resolves when the file is fully downloaded.
You can fix that with this refactored version:
const download_audio_file = async (url, filepath) => {
const promise = new Promise((resolve, reject) => {
https.get(url, (res) => {
const writeStream = fs.createWriteStream(filepath);
res.pipe(writeStream);
writeStream.on("finish", () => {
writeStream.close();
console.log("File downloaded");
resolve();
});
writeStream.on("error", (err) => {
console.log(err);
reject(err);
});
});
});
return promise;
};
Secondly, you are using Promise.all which awaits for all the promises in parallel.
You can replace that code snippet with:
const req = https.get(URL_TO_PARSE, async (res) => {
let xml = '';
res.on('data', (stream) => {
xml = xml + stream;
});
res.on('end', () => {
parser.parseString(xml, async (err, result) => {
if (err) {
console.log(err);
} else {
let items = result.rss.channel[0].item;
for(const item of items) {
let title = item.title[0];
let enclosure = item.enclosure[0];
let url = enclosure.$.url;
let filepath = `./${title}`;
console.log(`Downloading ${title} to ${filepath}`);
await download_audio_file(url, filepath);
}
}
});
});
});
Notice how I replaced the Promise.all with for(const item of items)
On a Node.js GraphQL API, using express + #graphql-yoga/node + graphql-upload-minimal, I have the uploads working very well, but when I upload a huge file, and the upload file size limit is reached, the stream continues until be finished, and it imposes makes unnecessary waiting to finish the entire stream.
I tried the below code, but the 'reject' does't destroy the stream:
stream.on('limit', function () {
const incomplete_file = `${folder}/${my_filename}`;
fs.unlink(incomplete_file, function () {});
reject(
new GraphQLYogaError(`Error: Upload file size overflow`, {
code: 'UPLOAD_SIZE_LIMIT_OVERFLOW',
}),
);
});
Full module below:
import getFilename from './getFilename';
import fs from 'fs';
import { GraphQLYogaError } from '#graphql-yoga/node';
export default async function uploadSingleFile(
folder: string,
file: any,
): Promise<any> {
return new Promise(async (resolve, reject) => {
const { createReadStream, filename, fieldName, mimetype, encoding } =
await file;
const my_filename = getFilename(filename, fieldName);
let size = 0;
let stream = createReadStream();
stream.on('data', function (chunk: any) {
size += chunk.length;
fs.appendFile(`${folder}/${my_filename}`, chunk, function (err) {
if (err) throw err;
});
});
stream.on('close', function () {
resolve({
nameOriginal: filename,
nameUploaded: my_filename,
mimetype: mimetype,
});
});
stream.on('limit', function () {
const incomplete_file = `${folder}/${my_filename}`;
fs.unlink(incomplete_file, function () {});
reject(
new GraphQLYogaError(`Error: Upload file size overflow`, {
code: 'UPLOAD_SIZE_LIMIT_OVERFLOW',
}),
);
});
})
.then((data) => data)
.catch((e) => {
throw new GraphQLYogaError(e.message);
});
}
How I can force imediate end of stream? There is any method to destroy the stream?
Thanks for the help!
I am trying to make resizing of a photo and uploading to a server synchronous. I believe the key will be making the stream function synchronous.
function upload(photo, newname, size, cb) {
gm(request(photo))
.resize(size[0], size[1])
.stream(function(err, stdout, stderr) {
var buf = new Buffer('');
stdout.on('data', function(data) {
buf = Buffer.concat([buf, data]);
});
stdout.on('end', function(data) {
var data = {
Bucket: config.s3_bucket_photos,
Key: newname,
Body: buf,
ACL: 'public-read',
ContentType: "image/jpeg"
};
uploadToServerSynchronous(data, cb);
});
});
}
It's not good to do sync operations in NodeJS - it stops processing event queue while Your sync method ends it's work.
As I understand You name "synchronous" things that has callbacks.
How about converting stream to promise?
const downloadAndResizePhotoByUrl = (photoUrl, size) => {
return new Promise((resolve, reject) => {
gm(request(photoUrl))
.resize(size[0], size[1])
.streamFn((error, stdout, stderr) => {
if(error) return reject(error);
let buffer = Buffer.from('');
stdout.on('data', data => {
buffer = Buffer.concat([buffer, data]);
});
stdout.on('end', () => resolve(buffer));
stdout.on('error', error => reject(error));
});
});
}
const uploadPhotoFromUrlToS3 = async (photoUrl, uploadAs, size, cb) => {
try {
const data = await downloadAndResizePhotoByUrl(photoUrl, size);
uploadToServerSynchronous(
{
Bucket: config.s3_bucket_photos,
Key: uploadAs,
Body: Buffer.from(data),
ACL: 'public-read',
ContentType: "image/jpeg"
},
cb
);
}
catch (error) {
cb(error);
}
}
I'm using the Busboy to parse multipart/form-data in my server, and I want to store each file in a Buffer without automatically convert to utf8. Is it possible?
const result = { files: [] }
const busboy = new Busboy({
headers: req.headers
})
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
const temp = {}
file.on('data', (data) => {
temp.file += data
})
file.on('end', () => {
temp.filename = filename
temp.contentType = mimetype
result.files = [...result.files, temp]
})
})
busboy.on('field', (fieldname, value) => {
result[fieldname] = value
})
busboy.on('error', (error) => {
console.error(error)
})
Currently the file.on('data') doesn't work properly, I'm loosing information because the operation += automatically converts the buffer to utf8.
You can set temp.file to be an array instead of a string and concat the buffer array in the end.
busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
const temp = {file: []}
file.on('data', (data) => {
temp.file.push(data)
})
file.on('end', () => {
temp.file = Buffer.concat(temp.file)
temp.filename = filename
temp.contentType = mimetype
result.files = [...result.files, temp]
})
})