Mongoose/Nodejs validation error handling, memory leak issue - node.js

I am running into a case where nodejs seems to be leaking memory when mongoose objects are attempted be created, but fail because of validation errors. It seems strange to me, and I am seeking help from the community in getting to the bottom of it.
In the sample code where I've tried to recreate my problem, I have a very simple Mongoose model which validates one of its fields to be non-negative number. Repeated attempts at creation of invalid objects (with XVALUE as -1 in the code below), I see memory growing forever. But valid objects (with XVALUE as +1, say) works as expected, without any leaks. I am tracking memory using process.memoryUsage().heapUsed variable.
How do I get rid of the memory leak when attempts are made to create invalid objects? If I am using mongoose incorrectly, any suggestions on improving the usage would be much appreciated.
Versions: mongoose - 3.8.x and nodejs - 0.10.x.
The following is the sample code I have used to recreate my problem:
(function() {
var MAX_ITER_COUNT, MyModel, MyStuff, Schema, XVALUE, addEntry, conn, iterCount, mongoose;
mongoose = require("mongoose");
conn = mongoose.createConnection('mongodb://#127.0.0.1:27017/memtest');
conn.on("error", function(err) {
return console.error(err);
});
Schema = mongoose.Schema;
MyStuff = new Schema({
x: {
type: Number
}
});
XVALUE = -1;
MyStuff.path("x").validate((function(val) {
if (val < 0) {
return false;
}
return true;
}));
MyModel = conn.model("MyStuff", MyStuff, "stuff");
iterCount = 0;
MAX_ITER_COUNT = 100 * 1000;
addEntry = function() {
var currentMem, x;
currentMem = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
console.log("" + iterCount + " - memory is: " + currentMem + " MB");
x = new MyModel({
x: XVALUE
});
return x.save(function(err) {
if (err) {
console.log("" + iterCount + " - failed");
} else {
console.log("" + iterCount + " - Save successful");
}
if (iterCount < MAX_ITER_COUNT) {
addEntry();
}
return iterCount++;
});
};
conn.on("open", addEntry);
}).call(this);

What's happening is that with an invalid document, the x.save async operation completes immediately, calling its callback as soon as its parent addEntry function call completes. This leaves no chance for garbage collection to run and the memory usage keeps growing.
To fix this, put the recursive addEntry call within a call to setImmediate to give GC a chance to run between iterations:
addEntry = function() {
var currentMem, x;
currentMem = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
console.log("" + iterCount + " - memory is: " + currentMem + " MB");
x = new MyModel({
x: XVALUE
});
return x.save(function(err) {
if (err) {
console.log("" + iterCount + " - failed");
} else {
console.log("" + iterCount + " - Save successful");
}
if (iterCount < MAX_ITER_COUNT) {
setImmediate(addEntry);
}
return iterCount++;
});
};

Related

getFileAsync causing Excel to crash

I'm building an office-js add-in for Excel. I need to upload the current workbook to a back end server. I've implemented an example from the Micrsoft Documentation, which seems to work fine the first time I call it, but on subsequent calls, it causes Excel to crash. I'm using Excel 365 version 1812 (build 11126.20132)
Here is the link to the example in the MS docs:
https://learn.microsoft.com/en-us/javascript/api/office/office.document
There are many examples on this page, to find the one I'm working from search for "The following example gets the document in Office Open XML" I've included the example below for ease of reference.
The code just get's the current file and dumps the characters to the console's log. It works fine the first but crashes Excel the second time--after it has shown the length of FileContent.
export function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
console.log("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}else {
console.log("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
// console.log("file part",sliceResult.value.data)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
console.log("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
console.log("fileContent.length",fileContent.length)
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
Here is the result
File size:21489 #Slices: 1
fileContent.length 21489
Original example from Microsoft documentation (https://learn.microsoft.com/en-us/javascript/api/office/office.document)
// The following example gets the document in Office Open XML ("compressed") format in 65536 bytes (64 KB) slices.
// Note: The implementation of app.showNotification in this example is from the Visual Studio template for Office Add-ins.
function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
else {
app.showNotification("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
app.showNotification("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
// The following example gets the document in PDF format.
Office.context.document.getFileAsync(Office.FileType.Pdf,
function(result) {
if (result.status == "succeeded") {
var myFile = result.value;
var sliceCount = myFile.sliceCount;
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Now, you can call getSliceAsync to download the files,
// as described in the previous code segment (compressed format).
myFile.closeAsync();
}
else {
app.showNotification("Error:", result.error.message);
}
}
);
Since you're using Excel, have you tried the CreateWorkbork API? Might be a good workaround if the Document API has a bug, like Xuanzhou indicated earlier.
Here's a CreateDocument snippet that you can load into Script Lab. It shows how to create a Workbook copy based on an existing file.
Hope all that is helpful.
We already have a fix for it now. But the fix still need some time to go to production. Please try it several days later and let me know if the issue still exists. Thanks.

Excel js api large Data

I am currently working on an Excel web Add-in where I am making an ajax call which sometimes returning a very large amount of data. After the data is returned I am iterating thought the data and loading values to cells in one sync. If the returned data is big enough it would crash the add-in. I have tried loading the data to the cells in batches, meaning for every 500 rows I tried to sync and then continue loading data into cells, but after loading the first 500 rows instead of continuing after the sync it exits out of the loop. I am new to Excel Js API and I am not sure I am doing this portion of my code correctly and I am unable to find any examples of this, any help would be appreciated.
function loadExcelData() {
Excel.run(function (context) {
var WebConnectAPI = "../../Data/GetExcelData";
$("#Info").html("")
var app = context.workbook.application;
app.load("calculationMode");
return context.sync()
.then(function () {
app.suspendApiCalculationUntilNextSync();
$.when(getExcelData(WebConnectAPI)).done(function (data) {
LoadDataV3(data, context);
}).fail(function (jqXHR, textStatus, errorThrown) {
$("#Error").html("Error:" + textStatus);
console.log("jqXHr:" + jqXHR + " Status:" + textStatus + " error:" + errorThrown);
});
});
}).catch (function (error) {
console.log("Error: " + error);
$("#Error").html(error);
if (error instanceof OfficeExtension.Error) {
console.log("Debug info: " + JSON.stringify(error.debugInfo));
}
});
function LoadDataV3(data, context) {
var workSheetName = "Sheet1";
var currentWorksheet = context.workbook.worksheets.getItem(workSheetName);
if (data.data.length != 0) {
for (var x = 0; x < data.data.length; x++) {
var y = 0
if (x % 500 == 0 && x != 0) {
context.sync();
loadCellData(currentWorksheet, data.data, x, y);
}
else {
loadCellData(currentWorksheet, data.data, x, y);
}
}
return context.sync()
.then(function () {
$('#Export-Data').show();
});
}
else {
$("#Info").html("No data was returned for your specific search criteria.")
}
}
function loadCellData(currentWorksheet,excelData,x,y) {
$.each(excelData[x], function (key, value) {
if (x == 0) {
var HeaderCell = currentWorksheet.getRange(tocellAddress(y + 1, x + 1));//Start on first row
var Cell = currentWorksheet.getRange(tocellAddress(y + 1, x + 2));//Start on second row
HeaderCell.values = [[key]];
Cell.values = [[value]];
HeaderCell.format.autofitColumns();
Cell.format.autofitColumns();
}
else {
var Cell = currentWorksheet.getRange(tocellAddress(y + 1, x + 2));//start on third row
Cell.values = [[value]];
Cell.format.autofitColumns();
}
y++;
});
}
The approach I took in my above code was not correct due to my misunderstanding of how the context.sync actually worked, as mentioned in the js api documentation you should really just need one context sync. For loading large amounts of data the data should be queued up in chunks of ranges and then sync once all data is queued.

Run an event only once in NodeJS but use the data multiple times

So I have written this code :
manager.getInventoryContents(730, 2, true, (err,inventory,currencies) => {
if (err){
console.log(err)
} else {
console.log("Create order for : " + orderitemname.length + " items.")
var otn = orderitemname.length -1;
while (otn !== -1) {
var li= inventory.length - 1;
while (li !== -1){
if (inventory[li].market_name === orderitemname[otn]){
console.log("Add item to trade " + orderitemname[otn]);
li = li -1;
otn = otn -1;
} else {
console.log("ERR !!! ITEM NOT IN INVENTORY !!! " + orderitemname[otn]);
//Change Order Status To Failed !!
n = n-1;
otn = -1;
li = -1;
}
}
}
}
})
So what is happening is that the process is calling this event multiple times (since it is in a while loop), and the node module is ignoreing it saying :
Error: The request is a duplicate and the action has already occurred in the past, ignored this time
So is there a way I can just call the event once, save it to a variable or something and then use the data multiple time ?
You can wrap everything in a new context and initiate whatever variable you need there. Then, the variable will be accessible from within your callback function.
Something like this:
(() => {
let ctx_data = 0;
manager.getInventoryContents(730, 2, true, (err,inventory,currencies) => {
console.log(++ctx_data);
});
})();

fabricjs on retina: new object jumps to left top

I continue my work on collaborative sketch tool and trying to add retina devices support. Currently i have following behavior if user creating drawing on ipad air:
small movie
Here is my code:
this.getZoomLevel = function (height) {
if (height > 1024) {
return 1024 / height;
} else {
return height / 1024;
}
};
this.calculateCanvasSize = function(pHeight, pWidth) {
var result = {
height: 0,
width: 0
};
while (result.width < pWidth - 1 && result.height < pHeight - 1) {
result.height = result.height + 1;
result.width = result.height * 4 / 3;
}
return result;
};
this.initCanvas = function () {
try {
var parent = document.getElementsByClassName('komaso-canvas-container')[0];
var canvasSize = this.calculateCanvasSize(parent.clientHeight, parent.clientWidth);
var canvasHtml = "<div id='wrapper-" + this.Id + "' class='whiteboard-canvas-wrapper' data-ng-show='CurrentPage.Id==" + this.Id + "'><canvas width='" + canvasSize.width + "' height='" + canvasSize.height + "' id='whiteboard-" + this.Id + "' class='whiteboard'><p>Your brower does not support Canvas/p></canvas></div>";
$(parent).append($compile(canvasHtml)(scope));
this.Canvaso = document.getElementById(this.HtmlId);
if (!this.Canvaso) {
console.log('Error: Cannot find the imageView canvas element!');
return;
}
if (!this.Canvaso.getContext) {
console.log('Error: no canvas.getContext!');
return;
}
this.FabricCanvas = new fabric.Canvas(this.HtmlId, { selectionColor: 'transparent' });
this.FabricCanvas.setWidth(canvasSize.width);
this.FabricCanvas.setHeight(canvasSize.height);
fabric.Object.prototype.transparentCorners = false;
this.FabricCanvas.on('mouse:down', this.onMouseDown);
this.FabricCanvas.on('mouse:up', this.onMouseUp);
this.FabricCanvas.on('mouse:move', this.onMouseMove);
this.FabricCanvas.on('object:added', this.onObjectAdded);
this.FabricCanvas.on('text:editing:exited', self.onTextObjectEdited);
if (window.devicePixelRatio !== 1) {
var c = this.FabricCanvas.getElement();
var w = c.width, h = c.height;
c.setAttribute('width', w * window.devicePixelRatio);
c.setAttribute('height', h * window.devicePixelRatio);
$(c).width(canvasSize.width);
$(c).height(canvasSize.height);
c.getContext('2d').scale(window.devicePixelRatio, window.devicePixelRatio);
}
this.FabricCanvas.setZoom(this.getZoomLevel(this.Canvaso.height));
this.ToggleTool(self.CurrentTool.ToolName);
this.WhiteboardInitiated = true;
} catch (e) {
console.log(e);
}
};
getZoomLevel returns value to pass into SetZoom method of fabric js canvas object. We decided to have all clients canvas aspects are 4:3 and default dimension is 1024*768. So based on this dimensions we calculation zoom factor.
calculateCanvasSize - returns width and height for canvas according to 4:3 rule.
If you have any idea about how to fix this wrong behavior then post your comment please. Thank you in advance!
I would suggest you yo update to a retina enabled version of fabricjs (grab 1.6.2).
If, for any reason you can't, i think the problem is here:
if (window.devicePixelRatio !== 1) {
var c = this.FabricCanvas.getElement();
...
c.getContext('2d').scale(window.devicePixelRatio, window.devicePixelRatio);
}
getContext return a new context. This is not the context where fabric is gonna render later. If you want to have retina enabled lowerCanvas you have to scale this.FabricCanvas.contextContainer that gets created and referenced on fabric.Canvas initialization.
I suggest you to switch to newer fabric anyway.

Error when using fiber dependent wait.for npm module in Node.js

Im using the wait.for library to get rid of some asynchronous calls. Im doing this because I don't really care much about having my server run efficiently or anything, I'm using node.js for some data processing using the Natural NLP npm module.
Anyways, Im recursively building a tree and requesting an wikipedia article for each node. That wikipedia request is asynchronous and I am trying to make it synchronous via a call to wait.for(...). The request works fine if the recursive call ends after just one activation, but throws an error if the recursion carries on any further.
Visually, I mean
[ROOT]
/ | \
[] [] [] <--- This level is fine ... No Errors
|
[] <--- Error Here, on level 2
The error is:
/Users/me/Documents/Node Projects/Personal Website/node_modules/wait.for/waitfor.js:37
fiber.run(); //resume fiber after "yield"
^
TypeError: Cannot call method 'toString' of undefined
at TfIdf.tfidf (/Users/me/Documents/Node Projects/Personal Website/node_modules/natural/lib/natural/tfidf/tfidf.js:132:42)
at isIrrelevant (/Users/me/Documents/Node Projects/Personal Website/finalproject/wikitree.js:126:47)
Here is the code for the recursive call:
var WikiTree = function(node) {
this.name = node.name;
this.context = node.context;
this.text = node.article;
if (node.children) {
this.children = node.children;
}
function createArticleSummary(article) {
// TODO: Must Implement
return article;
}
function isIrrelevant(article, title, linkText) {
// Tweak Here
var articleTfIdf = new TfIdf();
articleTfIdf.addDocument(article);
var titleRelevanceToArticle = articleTfIdf.tfidf(title,0); // ERROR HERE: This is wikitree.js:126
var textRelevanceToArticle = articleTfIdf.tfidf(linkText,0);
var titleRelevanceToZen = zenTfidf.tfidf(title,0);
var textRelevanceToZen = zenTfidf.tfidf(linkText,0);
return ( titleRelevanceToZen / titleRelevanceToArticle >= 1 ) || ( textRelevanceToZen / textRelevanceToArticle >= 1 );
}
WikiTree.prototype.searchChild = function(child) {
console.log("Searching for Link Title \"" + child.title + "\" And Link Text \"" + child.linkText + "\"");
var childQuery = {
query : child.title,
format : "html",
summaryOnly : false
};
var childArticle = wait.for(wikipedia.searchArticle, childQuery);
console.log("Got Wikipedia Response for " + child.title);
var childWikiLinks = extractLinkedWikis(childArticle);
console.log("Found " + childWikiLinks.length + " Links for " + child.title);
for (var i = childWikiLinks.length - 1; i >= 0; i--) {
if ( LinkTrie.contains(childWikiLinks[i].title) || LinkTrie.contains(childWikiLinks[i].linkText) ) {
childWikiLinks.splice(i,1);
} else if ( isIrrelevant( childWikiLinks[i] ) ) {
childWikiLinks.splice(i,1);
} else {
LinkTrie.addStrings([ childWikiLinks[i].title, childWikiLinks[i].text ]);
}
};
console.log("After pruning, " + childWikiLinks.length + " Links left for " + child.title);
var newChildWT = new WikiTree({
name : child.title,
context : child.linkText,
article : childArticle,
children : createArticleSummary(childWikiLinks)
});
return newChildWT;
};
WikiTree.prototype.descend = function() {
// Descend to the next level ... Meaning find the children of the current children (If there are any)
if (!this.children) {
console.log("Leaf at " + this.name);
return;
}
for (var i = this.children.length - 1; i >= 0; i--) {
if ( this.children[i].title.match(/Category:/) ) {
console.log("Found 'Category:' at " + i + " in " + this.children[i].title);
this.children.splice(i,1);
} else
this.children[i] = this.searchChild(this.children[i]);
};
console.log("Done With Descend For " + this.name);
};
this.descend();
};
var result = ...; // A String
var zenLinks = ...; // An Array of objects
// Actually make the first recursive call here
var zenTree = new WikiTree({
name : 'Zen & The Art of Motorcycle Maintenance',
article : result,
children : zenLinks
});
Anyways, any help will be much appreciated. There wasn't much help from googling the issue so anyone that has had experience with either wait.for or node fibers could greatly help me out.
I think you might do this:
var childArticle = wait.forMethod(wikipedia, 'searchArticle', childQuery);

Resources