Hello I am taking JSON passing it through zlib for compressing, storing it as a buffer in a flat file database, and then reading it and sending it out.
Except my problem is the data is all sorts of crazy characters. I've tried .toString() Ive tried the official StringDecoder from Node. I've tried a lot of things but I can't seem to get it in any format other then the .toJSON readable as a JSON buffer when I actually need the outputted final JSON.
Ideas?
WRITING TO FLAT FILE DB
export const writeCollection = (index, timespan, data) => {
zlib.gzip(JSON.stringify(data), (err, result) => {
if (err) {
console.log({ err });
} else {
const keyName = dbCollections.gzip[index].add({
result
});
collectionKeys.gzip[index] = keyName;
writeLogging(timespan, keyName, index, "gzip");
}
});
zlib.brotliCompress(JSON.stringify(data), (err, result) => {
if (err) {
console.log({ err });
} else {
const keyName = dbCollections.brotli[index].add({
result
});
collectionKeys.brotli[index] = keyName;
writeLogging(timespan, keyName, index, "brotli");
}
});
};
READING FROM FLAT FILE DB
export const readCollection = (index, encoding) => {
const encodedRead = encoding.includes("br")
? dbCollections.brotli[index].all()
: dbCollections.gzip[index].all();
return encodedRead[0].result;
};
TRYING TO CONVERT TO JSON
export const testGetQuakeData = (req, res) => {
const encoding = req.headers["accept-encoding"];
try {
const data = readCollection(0, encoding);
console.log(data)
const json = decoder.write(Buffer.from(data));
console.log(json)
// res.set({
// 'Content-Type': 'application/json',
// 'Content-Encoding': encoding.includes('br') ? 'br' : "gzip",
// })
res.send(json)
} catch (err) {
console.log({ err });
res.status(500).send(err);
}
};
FORGOT TO UNZIP WITH ZLIB!
export const testGetQuakeData = (req, res) => {
const encoding = req.headers["accept-encoding"];
try {
const data = readCollection(0, encoding);
encoding.includes("br")
? zlib.brotliDecompress(data, (err, result) => {
err ? res.status(500).send(err) : res.send(decoder.write(result));
})
: zlib.unzip(data, (err, result) => {
err ? res.status(500).send(err) : res.send(decoder.write(result));
});
} catch (err) {
console.log({ err });
res.status(500).send(err);
}
};
HERES THE FUTURE PROD FUNCTION!
export const testGetCompressedQuakeData = (req, res) => {
const encoding = req.headers["accept-encoding"];
try {
const data = readCollection(0, encoding);
encoding.includes("br")
? res.writeHead(200, {
"Content-Type": "application/json",
"Content-Encoding": "br",
"Content-Length": data.length,
})
: res.writeHead(200, {
"Content-Type": "application/json",
"Content-Encoding": "gzip",
"Content-Length": data.length,
})
res.end(data)
} catch (err) {
console.log({ err });
res.status(500).send(err);
}
};
Content Length 6319 (Brotli) vs 63148 (JSON)
Brotli Compression for the win!
Related
I read Pipe a stream to s3.upload()
but im having difficulty with I am not sure if that actually solves and I have tried.
What I am doing is a get call to www.example.com. this returns a stream, I want to upload that stream to s3.
heres my try.
fetch('https://www.example.com',fileName{
method: 'GET',
headers: {
'Authorization': "Bearer " + myAccessToken,
},
})
.then(function(response) {
return response.text();
})
.then(function(data) {
uploadToS3(data)
});
const uploadToS3 = (data) => {
// Setting up S3 upload parameters
const params = {
Bucket:myBucket,
Key: "fileName",
Body: data
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
output: ///File uploaded successfully. https://exampleBucket.s3.amazonaws.com/fileName.pdf
however this is blank.
I figured it out, but i did not keep using fetch.
and I actually download the file, then upload it. then delete the file.
function getNewFilesFromExampleDotCom(myAccessToken, fileName, fileKey) {
let url2 = 'https://example.com' + fileKey;
axios
.get(url2, {
headers: { 'Authorization': "Bearer " + myAccessToken },
responseType: 'stream',
})
.then(response => {
let file = fileName;
response.data.pipe(fs.createWriteStream(file))
let myFileInfo = [];
if( myFileInfo.length > 0){
myFileInfo.splice(0, myFileInfo.length)
}
myFileInfo.push(file)
processArray(myFileInfo)
console.log(file + " saved")
})
.catch(error => console.log(error));
}
async function processArray(array) {
for (const item of array) {
await delayedLog(item);
}
console.log('Downloaded!');
console.log('Uploading to s3!');
}
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function delayedLog(item) {
await delay();
uploadFiles(item)
}
async function uploadFiles(file){
uploadToS3List(file)
await new Promise((resolve, reject) => setTimeout(resolve, 1000));
deleteMyFiles(file)
}
const uploadToS3List = (fileName) => {
// Read content from the file
const fileContent = fs.readFileSync(fileName);
// Setting up S3 upload parameters
const params = {
Bucket:"myBucketName",
Key: fileName,
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
function deleteMyFiles(path){
fs.unlink(path, (err) => {
console.log(path + " has been deleted")
if (err) {
console.error(err)
return
}
})
}
I am trying to upload an image file using mutler along with some form data. Every solution I've looked suggests to disable the bodyparser but if I disable the body parser then I cannot parse the request body.
Backend POST api along with function that runs middleware:
case 'POST':
if (req.body instanceof FormData) {
await runMiddleware(req, res, upload.single('image'))
}
console.log('ID: ',req.query.id)
Courses.findByIdAndUpdate(req.query.id, req.body, {new: true}, (err, result) => {
if(err)
{
console.log('Error in findByIdAndUpdate: ',err)
res.status(400).json({ success: false, data: result })
}
else {
console.log('Success in findByIdAndUpdate: ',result)
res.status(200).json({ success: true, data: result })
}
})
below is the function
function runMiddleware(req, res, fn) {
return new Promise((resolve, reject) => {
fn(req, res, (result) => {
if (result instanceof Error) {
return reject(result)
}
return resolve(result)
})
})
}
Upload Middleware
import multer from "multer";
const storage = multer.diskStorage({
destination: (req, file, cb) => {
cb(null, "public/uploads");
},
filename: (req, file, cb) => {
cb(null, "upload" + Date.now() + "-" + file.originalname)
}
})
module.exports = multer({storage})
Front-End API call
const createUpdateFormData = () => {
const data = {
title: getValues('title'),
categories: getValues('categories'),
description: getValues('description'),
image: (getValues('file') ? getValues('file')[0] : null),
}
let formData;
if (data.image) {
formData = new FormData();
Object.entries(data).forEach(([key, value]) => {
if(value)
formData.append(key, value);
});
}else {
formData = {}
Object.entries(data).forEach(([key, value]) => {
if(value)
formData[key] = value;
});
}
return formData
}
const handleUpdateRequest = (id,formData) => {
console.log(formData)
const updateReq = axios.post(`/api/courses/${id}`, formData)
toast.promise(updateReq, {
loading: "Processing...",
error: (err) => {
console.log(err)
if (err.response.status === 409)
return 'Course already exists!'
else
return 'Oops something went wrong!'
},
success: (res) => {
console.log(res)
const newState = courseState.map(course => course._id === id ? res.data.data : course)
setCourses(newState)
// console.log('NewState: ', courseState)
return "Course updated successfully!"
}
});
}
I am using the useForm hook
It worked yesterday, and now it stopped without any changes made to the code. What is going on?
Client
async function uploadFile(file) {
let formData = new FormData();
formData.append("recordUid", recordUid);
formData.append("fieldUid", fieldUid);
formData.append("file", file);
await fetchPostFormData("/api/files", formData);
}
async function fetchPostFormData(url, formData) {);
try {
let result = await (
await fetch(url, {
method: "POST",
withCredentials: true,
credentials: "include",
headers: {
Authorization: localStorage.getItem("token"),
},
body: formData,
})
).json();
return result;
} catch (err) {
return err;
}
}
Server
router.post("/api/files", async (req, res, next) => {
try {
console.log("starting upload..."); // <------------------- THIS ONE IS LOGGED
let bb = busboy({
headers: req.headers,
limits: {
fileSize: 20 * 1024 * 1024, // 20 mb
},
});
let fields = {};
// Get any text values
bb.on("field", (fieldname, val, fieldnameTruncated, valTruncated) => {
console.log("on.field", fieldname, val); // <------------------ NOT FIRING
fields[fieldname] = val;
});
bb.on("file", (fieldname, file, filename, encoding, mimetype) => {
console.log("on.file"); // <----------------------------------- NOT FIRING
let parts = filename.filename.split(".");
let name = parts[0];
let extension = parts[parts.length - 1];
let finalName = `${+new Date()}-${name}.${extension}`;
let filePath = `${filesFolderPath}${finalName}`;
// Open writeable stream to path
let writeStream = fs.createWriteStream(filePath);
// Pipe the file to the opened stream
file.pipe(writeStream);
// Check for errors
writeStream.on("error", (err) => {
console.log(err);
});
writeStream.on("close", async (err) => {
let sizeBytes = fs.statSync(filePath).size;
});
});
bb.on("finish", () => {
res.status(200).send({ success: true });
});
} catch (err) {
next(err);
}
});
Managed to solve it.
The problem was the missing req.pipe(bb) at the very end.
// previous code... ^^^^^
bb.on("finish", () => {
res.status(200).send({ success: true });
});
req.pipe(bb) // <------------- THIS SHIT RIGHT HERE
} catch (err) {
next(err);
}
});
How to fix this error message?
[xhr.js?b50d:178 POST http://localhost:3000/editor/add net::ERR_CONNECTION_RESET][1]
It works and append data, but I get this error message...
My API looks like this:
app.js
app.post('/editor/add', function (req, res) {
let articleData;
let textData;
let article = {
title: req.body.title,
content: req.body.content
}
fs.readFile(urlPath, 'utf8', (err, data) => {
if (err) {
console.log('readfile => ' + err);
} else {
articleData = JSON.parse(data);
articleData[article.title] = article.content;
textData = JSON.stringify(articleData, null, 2);
fs.writeFile(urlPath, textData, 'utf8', (err) => {
if (err) {
console.log('write file => ' + err);
} else {
console.log('Finished writing');
}
});
}
});
});
And my Axios POST method looks like this.
editor.vue
submitEditor: function() {
var self = this;
self.$axios({
headers: {
"Content-Type": "application/json"
},
method: "post",
url: "http://localhost:3000/editor/add",
data: {
title: "test5",
content: self.html
}
})
.then(res => {
console.log(res);
})
.catch(error => {
if (!error.response) {
// network error
this.errorStatus = "Error: Network Error";
} else {
this.errorStatus = error.response.data.message;
}
});
}
I use Vue/cli, I separate my client code and my server code. They are on a separate folder. I put Vue/cli in my client folder, and express.js in my server folder.
Thank you in advance!
Try sending a response from your route:
fs.writeFile(urlPath, textData, 'utf8', (err) => {
if (err) {
console.log('write file => ' + err);
} else {
console.log('Finished writing');
res.json({ msg: 'success' }); // send the client something
}
});
I am very new to Node js, I just want to get the data from extenal xml from a website but I got an error from Firebase Function log TypeError: invalid media type. I think it come from when I try to do this task parseString(xml, function(err, result) { })
Anyone can help me, it will be great:
Here is my code on firebase function:
exports.getRate = functions.https.onRequest((req, res) => {
getRate = () => {
var url = "https://www.vietcombank.com.vn/ExchangeRates/ExrateXML.aspx";
https.get(url, function(res) {
var xml = "";
res.on('error', function(error){
console.log(error, 'get data error');
})
res.on("data", function(chunk) {
xml += chunk;
console.log(xml, 'xml file');
});
res.on("end", function() {
var date = "";
let rateAUD = {
code: 'AUD/VND',
buy: 0,
sell: 0
};
let rateUSD = {
code: 'USD/VND',
buy: 0,
sell: 0
};
parseString(xml, function(err, result) {
console.log(xml, 'xml file');
date = result.ExrateList.DateTime[0];
if (result.ExrateList.Exrate[0].$.CurrencyCode == "AUD") {
rateAUD.buy = result.ExrateList.Exrate[0].$.Buy;
rateAUD.sell = result.ExrateList.Exrate[0].$.Sell;
} else {
console.log("They change the database list");
}
if (result.ExrateList.Exrate[18].$.CurrencyCode == "USD") {
rateUSD.buy = result.ExrateList.Exrate[18].$.Buy;
rateUSD.sell = result.ExrateList.Exrate[18].$.Sell;
} else {
console.log("They change the database list");
}
console.log(rateAUD, rateUSD, 'get data');
uploadDataToServer(date, { rateAUD, rateUSD });
if(err) {
console.log(err);
}
});
});
});
};
function uploadDataToServer(date, { rateAUD, rateUSD }) {
var db = admin.firestore();
let data = { rateAUD, rateUSD };
data.timeStamp = date;
console.log('upload success');
db.collection("liveRate").add(data),then((err)=> {
console.log(err);
});
}
return res.status(200)
.type('application / json')
.send('hello')
});
'
When I run the same code on another Nodejs playground, it works well.
Here is the link: https://repl.it/repls/MaroonSlateblueProfiler
So weird!
Ps: my payment option is ON.
The problem is that the client is sending the server what may or may not be a valid media type in an encoding the server cannot understand (as per the Content-Encoding header the client packaged with the request message).
Please try to set the content-type to xml:
getRate = () => {
var options = {
hostname: "www.vietcombank.com.vn",
port: 443,
path: "/ExchangeRates/ExrateXML.aspx",
headers: {
'Content-Type': 'application/xml'
}
};
https.get(options, function(res) {
...
});
}