I have been at this problem for days now and still can't find out what the problem must be.
The HTTPClient is not being cleaned up properly and thus the apps memory is exploding when looping over say 100 images to download.
I am testing in a simple alloy template created by using appc new -t titanium.
only an index view is opened with one button to start the download loop
my index file looks like the following:
var max = 100
var i = 0
var c = Ti.Network.createHTTPClient();
c.onerror = function () {
c = null;
}
c.onload = function () {
saveFile(this.responseData)
console.log("done get")
i++
getfiles()
}
function getfiles () {
if (i < max) {
try {
c.open('GET', "https://cdn.fossilswitzerland.ch/large0/FS4931.jpg");
c.send();
} catch (e) {
}
} else {
c = null
console.log("finished")
}
}
function saveFile (response) {
console.log("STORING")
var f = Ti.Filesystem.getFile(Ti.Filesystem.applicationDataDirectory, 'FS4931.jpg');
f.write(response);
if (f.exists() === false) {
console.log("NOT STORED")
} else {
console.log("STORED")
}
f = null;
}
$.index.open();
Initially after the app has booted memory is at ca. 30mb
So far so good
After clicking get images button on the index page, memory goes up to ca. 62mb
Then after a while goes down to ca. 50mb which leaves 20mb unaccounted for
I am using:
Xcode 10.3
Ti SDK Version 8.1.1.GA
Simulator: iPad Pro (2nd generation) on 12.4
When changing max to 1000, memory soars to around 322mb and doesn't seem to decrease at any more.
This is a really big problem when looping through and downloading say 2000 images because the app just crashes at some point
Thank you for any help
I've changed the code a bit (for-loop, use the file property, don't re-use the httpclient (check the documentation), classic code so it is quicker to test):
var w = Ti.UI.createWindow();
var b = Ti.UI.createButton({
title: "download"
});
w.add(b);
function onError(e) {
console.log("error", e);
}
function onLoad(e) {
console.log("done get")
}
function getfiles(e) {
for (var i = 0; i < 100; ++i) {
var c = Ti.Network.createHTTPClient({
onerror: onError,
onload: onLoad
});
c.open('GET', "https://cdn.fossilswitzerland.ch/large0/FS4931.jpg");
var fname = Ti.Filesystem.applicationSupportDirectory + "/" + (new Date().getTime()) + ".jpg";
console.log("Download to", fname);
c.file = fname;
c.send();
c = null;
fname = null;
}
}
b.addEventListener("click", getfiles);
w.open();
I get a little overhead when it is releasing memory (around 5mb). You could create a JIRA ticket (https://jira.appcelerator.org/) so they can check it.
Related
I'm building an office-js add-in for Excel. I need to upload the current workbook to a back end server. I've implemented an example from the Micrsoft Documentation, which seems to work fine the first time I call it, but on subsequent calls, it causes Excel to crash. I'm using Excel 365 version 1812 (build 11126.20132)
Here is the link to the example in the MS docs:
https://learn.microsoft.com/en-us/javascript/api/office/office.document
There are many examples on this page, to find the one I'm working from search for "The following example gets the document in Office Open XML" I've included the example below for ease of reference.
The code just get's the current file and dumps the characters to the console's log. It works fine the first but crashes Excel the second time--after it has shown the length of FileContent.
export function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
console.log("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}else {
console.log("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
// console.log("file part",sliceResult.value.data)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
console.log("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
console.log("fileContent.length",fileContent.length)
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
Here is the result
File size:21489 #Slices: 1
fileContent.length 21489
Original example from Microsoft documentation (https://learn.microsoft.com/en-us/javascript/api/office/office.document)
// The following example gets the document in Office Open XML ("compressed") format in 65536 bytes (64 KB) slices.
// Note: The implementation of app.showNotification in this example is from the Visual Studio template for Office Add-ins.
function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
else {
app.showNotification("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
app.showNotification("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
// The following example gets the document in PDF format.
Office.context.document.getFileAsync(Office.FileType.Pdf,
function(result) {
if (result.status == "succeeded") {
var myFile = result.value;
var sliceCount = myFile.sliceCount;
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Now, you can call getSliceAsync to download the files,
// as described in the previous code segment (compressed format).
myFile.closeAsync();
}
else {
app.showNotification("Error:", result.error.message);
}
}
);
Since you're using Excel, have you tried the CreateWorkbork API? Might be a good workaround if the Document API has a bug, like Xuanzhou indicated earlier.
Here's a CreateDocument snippet that you can load into Script Lab. It shows how to create a Workbook copy based on an existing file.
Hope all that is helpful.
We already have a fix for it now. But the fix still need some time to go to production. Please try it several days later and let me know if the issue still exists. Thanks.
Getting this warning on the following code:
workflow.removeZSets = function(fn) {
var processed = 0;
for (var c = 1; c < 10; c++) {
workflow.removeZSet(c, function() {
processed++;
if (processed === 9) {
return fn(null, "finished removing");
}
});
}
}
workflow.removeZSet = function(precision, fn) {
rc.zrem("userloc:" + precision, function() {
return fn(null, 'done');
});
});
}
Does anyone have a suggestion how to accomplish this without triggering the warning?
I have some ideas like using the async library to run them all in parallel but this is a fairly common thing I do throughout this code base so interested in feedback on the best way.
The error is because you have define a function within your for loop.
You could try something like this, defining the function outside of the loop:
workflow.removeZSets = function(fn) {
var processed = 0;
function removeZ(c) {
workflow.removeZSet(c, function(err) {
processed++;
if (processed === 9) {
return fn(null, "finished removing");
}
});
}
for (var c = 1; c < 10; c++) {
removeZ(c);
}
}
Using a library like async to do the looping would help clean up your code, it would allow you to avoid checking if all the items have processed (processed ===9) because it is handled by async.
In a chrome extension, you can set permissions in manifest.json. I assume crossrider generates this file behind the scenes. How can change the permissions generated by a crossrider extension or is it not possible yet?
My extension in particular needs access to image and video data on all pages. This gets denied with a CORS error and I believe setting the proper permissions would solve my problem.
[EDIT]
Here's the core of my code:
try {
//all nodes in the DOM go through this function
var parseNode = function(node) {
//only get img and video tags
var nodeName = node.nodeName.toUpperCase();
if (["IMG", "VIDEO"].indexOf(nodeName) == -1)
return;
//attempt to extract their pixel data
var canvas = document.createElement("canvas");
var context = canvas.getContext("2d");
try {
console.log(node.src, " ", node.clientWidth, "x", node.clientHeight);
canvas.width = node.clientWidth; //may be zero if failed CORS
canvas.height = node.clientHeight;
context.drawImage(node, 0, 0);
var dat = context.getImageData(0, 0, canvas.width, canvas.height);
console.log("Success");
canvas.remove();
return dat.pixels;
}
catch (e) {
console.log("Failed ", node, ": ", e);
canvas.remove();
}
return null;
};
//parse everything currently loaded
var everything = document.getElementsByTagName("*");
for (var i = 0; i < everything.length; i++) {
parseNode(everything[i]);
}
//use a mutation ovserver to parse everything that gets injected later
var parseMutations = function(mutations) {
mutations.forEach(function(mutation) {
if (mutation.addedNodes) {
for (var i = 0; i < mutation.addedNodes.length; i++) {
parseNode(mutation.addedNodes[i]);
}
}
});
};
var observer = new MutationObserver(parseMutations);
observer.observe(document, {
childList: true,
subtree: true
});
}
catch (e)
{
//this has to be here since all browsers are so shit at catching syntax errors in extensions
//not to mention the crossrider extension won't install properly if there's a typo or missing semicolon. so much pain
console.log(e, " ", e.stack); //stack is pretty useless with crossrider code injection
}
On many pages I just get a tonne of these:
DOMException {
code: 18
message: "Failed to execute 'getImageData' on 'CanvasRenderingContext2D': the canvas has been tainted by cross-origin data."
name: "SecurityError"
...
[EDIT]
I've removed the try/catch so the errors print properly. I'm still seeing lots of errors.
Uncaught SecurityError: Failed to execute 'getImageData' on 'CanvasRenderingContext2D': the canvas has been tainted by cross-origin data.
I'm using this page to test with: http://en.wikipedia.org/wiki/HSL_and_HSV
"Run in IFrame" is off.
try {
//all image and video nodes in the DOM go through this function
var parseNode = function(node) {
//attempt to extract their pixel data
var canvas = document.createElement("canvas");
var context = canvas.getContext("2d");
node.title = "FAILED";
console.log(node.src, " ", node.clientWidth, "x", node.clientHeight);
canvas.width = Math.max(1, node.clientWidth); //may be zero if failed CORS
canvas.height = Math.max(1, node.clientHeight);
context.drawImage(node, 0, 0);
var dat = context.getImageData(0, 0, canvas.width, canvas.height);
canvas.remove();
return dat.pixels;
node.title = "SUCCESS";
return null;
};
//parse everything currently loaded
var everything = document.getElementsByTagName("*");
for (var i = 0; i < everything.length; i++) {
var node = everything[i];
var nodeName = node.nodeName.toUpperCase();
if (["IMG", "VIDEO"].indexOf(nodeName) != -1)
(function(n) {setTimeout(function(){parseNode(n);},1000);})(node);
}
//use a mutation ovserver to parse everything that gets injected later
var parseMutations = function(mutations) {
mutations.forEach(function(mutation) {
if (mutation.addedNodes) {
for (var i = 0; i < mutation.addedNodes.length; i++) {
var node = mutation.addedNodes[i];
var nodeName = node.nodeName.toUpperCase();
if (["IMG", "VIDEO"].indexOf(nodeName) != -1)
(function(n) {setTimeout(function(){parseNode(n);},1000);})(node);
}
}
});
};
var observer = new MutationObserver(parseMutations);
observer.observe(document, {
childList: true,
subtree: true
});
}
catch (e)
{
//this has to be here since all browsers are so shit at catching syntax errors in extensions
console.log(e, " ", e.stack); //stack is pretty useless with crossrider code injection
}
Currently, the Crossrider platform does not provide a mechanism for modifying manifest permissions though there are plans to consider this for future releases.
Hence, you may manually experiment with adding the permission in the CRX file, though bear in mind that it may hinder the ability to support your extension.
[Disclosure: I am a Crossrider employee]
I use this function to watch an array of objects for changes:
$scope.$watch('Data', function (newVal) { /*...*/ }, true);
How can I get an object in which property has been changed so that I can push it in an array?
For example:
var myApp = angular.module("myApp", []);
myApp.factory("Data", function(){
var Data = [{id:1, property: "Random"}, {id:2, property: "Random again"}];
return Data;
});
var myBigArray = [];
function tableCtrl($scope, Data){
$scope.TheData = Data;
$scope.$watch("TheData", function() {
//Here an object should be pushed
myBigArray.push(">>Object in which property has been changed <<<");
}, true);
}
I don't see a way currently in Angular to get the changed object... I suspect you might need to traverse the new array and try to find the differences with the old array...
Edit: Note that this solution turns out to be a bad practice as it is adding a lot of watchers, which is something you do not want because it has a performance penalty.
=======
I eventually came up with this solution:
items.query(function (result) {
_(result).each(function (item, i) {
$scope.items.push(item);
$scope.$watch('items[' + i + ']' , function(){
console.log(item); // This is the item that changed.
}, true);
});
});
There is still no option like this for $watch, but you can use jQuery plugin for that, http://archive.plugins.jquery.com/project/jquery-diff
I implemented undo/redo with AngularJS using $watch, mb this can help
//History Manager Factory
.factory('HistoryManager', function () {
return function(scope) {
this.container = Array();
this.index = -1;
this.lock = false;
//Insert new step into array of steps
this.pushDo = function() {
//we make sure that we have real changes by converting to json,
//and getting rid of all hash changes
if(this.container.length == 0 || (angular.toJson(scope.widgetSlider) != angular.toJson(this.container[this.index][0]))) {
//check if current change didn't came from "undo" change'
if(this.lock) {
return;
}
//Cutting array, from current index, because of new change added
if(this.index < this.container.length-1) {
this.container = this.container.slice(0, this.index+1);
}
var currentStepSlider = angular.copy(scope.widgetSlider);
var selectedWidgetIndex = scope.widgetSlider.widgets.indexOf(scope.widgetCurrent);
//Initialising index, because of new "Do" added
this.index = this.container.length;
this.container.push([currentStepSlider, selectedWidgetIndex]);
if (this.onDo) {
this.onDo();
}
}
}
//Upon undo returns previous do
this.undo = function() {
this.lock = true;
if(this.index>0){
this.index--;
scope.widgetSlider = angular.copy(this.container[this.index][0]);
var selectedWidgetIndex = this.container[this.index][1];
scope.widgetCurrent = scope.widgetSlider.widgets[selectedWidgetIndex];
}
this.lock = false;
}
//Upon redo returns next do
this.redo = function() {
if(this.index < this.container.length-1) {
this.index++;
scope.widgetSlider = angular.copy(this.container[this.index][0]);
var selectedWidgetIndex = this.container[this.index][1];
scope.widgetCurrent = scope.widgetSlider.widgets[selectedWidgetIndex];
}
}
}
})
;
Please can you help me with me threading. I'm trying to download a file and at the same time update a BTProgressHUD progress display. I know that the reason that it's not working is to do with the download using the main thread and not allowing me to update the UI but I can't work out how to correctly use the thread pool to allow me to update the BTProgressHUD whilst the file is downloading. Please help!!
`
BTProgressHUD.Show("Downloading...", progress);
string this_file = "example.pdf";
string file_url = "http://our_server.com/files/" + this_file;
Uri url = new Uri(file_url);
var documents = Environment.GetFolderPath (Environment.SpecialFolder.MyDocuments);
var folder = Path.Combine (documents, "", "PDF");
System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
System.Net.HttpWebResponse response = (System.Net.HttpWebResponse)request.GetResponse();
response.Close();
Int64 iSize = response.ContentLength;
// keeps track of the total bytes downloaded so we can update the progress bar
Int64 iRunningByteTotal = 0;
// use the webclient object to download the file
using (System.Net.WebClient client = new System.Net.WebClient())
{
// open the file at the remote URL for reading
using (System.IO.Stream streamRemote = client.OpenRead(new Uri(file_url)))
{
// using the FileStream object, we can write the downloaded bytes to the file system
using (Stream streamLocal = new FileStream(folder + "/" + this_file, FileMode.Create, FileAccess.Write, FileShare.None))
{
// loop the stream and get the file into the byte buffer
int iByteSize = 0;
byte[] byteBuffer = new byte[iSize];
while ((iByteSize = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
// write the bytes to the file system at the file path specified
streamLocal.Write(byteBuffer, 0, iByteSize);
iRunningByteTotal += iByteSize;
// calculate the progress out of a base "100"
double dIndex = (double)(iRunningByteTotal);
double dTotal = (double)byteBuffer.Length;
double dProgressPercentage = (dIndex / dTotal);
int iProgressPercentage = (int)(dProgressPercentage * 100);
if (iProgressPercentage == 100)
{
var z = new UIAlertView ("Download Complete", this_file + " downloaded.", null, "OK", null);
z.Show();
BTProgressHUD.Dismiss();
}
if (iProgressPercentage % 10 == 0)
{
// THIS BUT NEVER HAPPENS!!! Cannot update the progress display
progress += 0.1f;
BTProgressHUD.Show("XXX", progress);
}
} // while..
streamLocal.Close(); // clean up the file stream
} // using stream
streamRemote.Close(); // close the connection to the remote server
} // using I.O
} // using system.net
`
Any help would be very very much appreciated.
I have used the TPL to kick of a background thread then called back to the UI by using InvokeOnMainThread. I have substituted the BTProgressHUD for a UILabel but it should work the same. Here is it working:
private void DownloadCoffeePDF()
{
Task.Factory.StartNew (() => {
InvokeOnMainThread(() => {
this.TheLabel.Text = string.Format("Downloading...{0}", progress);
});
string file_url = "http://www.pnf.org/coffeeedited041001.pdf";
Uri url = new Uri(file_url);
var documents = Environment.GetFolderPath (Environment.SpecialFolder.MyDocuments);
var folder = Path.Combine (documents, "", "PDF");
System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
System.Net.HttpWebResponse response = (System.Net.HttpWebResponse)request.GetResponse();
response.Close();
Int64 iSize = response.ContentLength;
// keeps track of the total bytes downloaded so we can update the progress bar
Int64 iRunningByteTotal = 0;
// use the webclient object to download the file
using (System.Net.WebClient client = new System.Net.WebClient())
{
// open the file at the remote URL for reading
using (System.IO.Stream streamRemote = client.OpenRead(new Uri(file_url)))
{
// using the FileStream object, we can write the downloaded bytes to the file system
using (Stream streamLocal = new FileStream(folder + "/" + "Coffee.pdf", FileMode.Create, FileAccess.Write, FileShare.None))
{
// loop the stream and get the file into the byte buffer
int iByteSize = 0;
byte[] byteBuffer = new byte[iSize];
while ((iByteSize = streamRemote.Read(byteBuffer, 0, byteBuffer.Length)) > 0)
{
// write the bytes to the file system at the file path specified
streamLocal.Write(byteBuffer, 0, iByteSize);
iRunningByteTotal += iByteSize;
// calculate the progress out of a base "100"
double dIndex = (double)(iRunningByteTotal);
double dTotal = (double)byteBuffer.Length;
double dProgressPercentage = (dIndex / dTotal);
int iProgressPercentage = (int)(dProgressPercentage * 100);
if (iProgressPercentage == 100)
{
InvokeOnMainThread(() => {
var z = new UIAlertView ("Download Complete", "Coffee.pdf" + " downloaded.", null, "OK", null);
z.Show();
this.TheLabel.Text = "Download Complete";
});
}
if (iProgressPercentage % 10 == 0)
{
InvokeOnMainThread(() => {
// THIS BUT NEVER HAPPENS!!! Cannot update the progress display
progress += 0.1f;
this.TheLabel.Text = string.Format("{0}", progress);
});
}
} // while..
streamLocal.Close(); // clean up the file stream
} // using stream
streamRemote.Close(); // close the connection to the remote server
} // using I.O
} // using system.net
});
}