I'm writing a Node.js extension in Rust. I called this toy library from C, checked it with Valgrind and no memory leak was found.
I called the same library from Python and Ruby, running in an infinite loop, and see no sign of a memory leak. (I can not post the picture due to insufficient reputation here).
When the same library is called from Node.js, the memory usage seems to increase with time:
The unit of time is the cycle of loop, not a real time.
Here is the Rust code:
#[repr(C)]
pub struct Matrix {
m: Vec<Vec<f64>>,
}
#[no_mangle]
pub extern "C" fn matrix_new(nrow: usize, ncol: usize) -> *mut Matrix {
let mut m = Vec::new();
for _ in 0..(nrow) {
let mut n = Vec::new();
for _ in 0..(ncol) {
n.push(0.0);
}
m.push(n);
}
Box::into_raw(Box::new(Matrix { m: m }))
}
#[no_mangle]
pub extern "C" fn matrix_free(matrix: *mut Matrix) {
if matrix.is_null() {
return
}
unsafe { Box::from_raw(matrix); }
}
Here is the original Node.js code:
var ref = require('ref');
var ffi = require('ffi');
var c_matrix = ref.types.void;
var c_matrix_ptr = ref.refType(c_matrix);
var libmatrix = ffi.Library('./target/release/libmatrix.so', {
'matrix_new': [ c_matrix_ptr, ['size_t', 'size_t']],
'matrix_get': [ 'double', [ c_matrix_ptr, 'size_t', 'size_t']],
'matrix_set': [ 'double', [ c_matrix_ptr, 'size_t', 'size_t', 'double']],
'matrix_free': [ 'void', [ c_matrix_ptr ]]
});
var matrix = function(nrow, ncol) {
"use strict";
var matrix = libmatrix.matrix_new(nrow, ncol);
Object.defineProperty(this, '_matrix', {
value: matrix,
writeable: false
});
return this;
};
matrix.prototype.get = function(row, col) {
"use strict";
return libmatrix.matrix_get(this._matrix, row, col);
};
matrix.prototype.set = function(row, col, value) {
"use strict";
libmatrix.matrix_set(this._matrix, row, col, value);
};
matrix.prototype.free = function() {
"use strict";
libmatrix.matrix_free(this._matrix);
};
module.exports = matrix;
if (!module.parent) {
while (true) {
var m = new matrix(3, 3);
m.free();
m = null;
delete global.m;
global.gc();
console.log(process.memoryUsage().rss);
}
}
Some information:
OS: Debian GNU/Linux (Jessie)
Node.js: 7.2.0
node-gyp: 7.2.0
ffi: 2.2.0
ref: 1.3.3
Rust: 1.13.0
I re-wrote the same library in C:
typedef struct Matrix {
size_t nrow;
size_t ncol;
double** mtx;
} Matrix;
void* matrix_new(size_t row, size_t col) {
Matrix* m = malloc(sizeof(Matrix));
m->nrow = row;
m->ncol = col;
m->mtx = malloc(m->nrow * sizeof(double*));
for (int i = 0; i < m->nrow; i++) {
*((m->mtx)+i) = (double*) malloc(m->ncol * sizeof(double));
}
for (int i = 0; i < m->nrow; i++) {
for (int j = 0; j < m->ncol; j++) {
m->mtx[i][j] = 0.0;
}
}
return (void*) m;
}
void matrix_free(void* m) {
if (m == NULL) {
return;
}
double** ptr = ((Matrix*) m)->mtx;
for (int i = 0; i < ((Matrix*) m)->nrow; i++) {
free((void*) (ptr[i]));
}
free((void*) ((Matrix*) m)->mtx);
free((void*) m);
}
I also changed the version of Node from 7.2.0 to 7.3.0. Here is the memory usage of the Node.js module with both C and Rust implementation:
I tried a no-op library without changing the Node.js code, finding something surprising myself:
Related
I am trying to monitor a binary file containing single 32 bit integer. I mapped the file into memory via MmapMut and read in a forever loop:
fn read(mmap: &MmapMut) {
let mut i = u32::from_ne_bytes(mmap[0..4].try_into().unwrap());
loop {
let j = u32::from_ne_bytes(mmap[0..4].try_into().unwrap());
if j != i {
assert!(false); // this assert should fail, but it never does
i = j;
}
}
}
However, compiler seems to optimise the loop away assuming neither i or j can ever change.
Is there a way to prevent this optimisation?
You can use MmapRaw with read_volatile().
fn read(mmap: &MmapRaw) {
let mut i = unsafe { mmap.as_ptr().cast::<u32>().read_volatile() };
loop {
let j = unsafe { mmap.as_ptr().cast::<u32>().read_volatile() };
if j != i {
assert!(false);
i = j;
}
}
}
I'm trying to re-implement this C code:
#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#define ROWS 100000
#define COLS 10000
int twodarray[ROWS][COLS];
int main() {
for (int i = 0; i < ROWS; i++) {
for (int j = 0; j < COLS; j++) {
twodarray[i][j] = rand();
}
}
int64 sum = 0;
for (int i = 0; i < ROWS; i++) {
for (int j = 0; j < COLS; j++) {
sum += twodarray[i][j];
}
}
}
So, after a lot of trial and error I've come up with Rust code that at least compiles
extern crate rand;
const ROWS: usize = 100000;
const COLS: usize = 10000;
fn main() {
let mut twodarray: [[u128; COLS]; ROWS] = [[0; COLS]; ROWS];
for i in 0..ROWS {
for j in 0..COLS {
twodarray[i][j] = rand::random::<u8>() as u128;
}
}
let mut sum: u128 = 0;
for k in 0..ROWS {
for j in 0..COLS {
sum += twodarray[k][j] as u128;
}
}
println!("{}", sum);
}
But when I compile and execute it, I get the following error message: thread 'main' has overflowed its stack. To be honest, I have absolutelly no idea why this is happening. I'm currently learning rust, but there is not a lot of information on 2d arrays online... I DO NOT WANT TO USE VECTOR. This exercise is specifically aimed on arrays.
EDIT:
After reading the accepted answear, I've come up with Rust code that outputs expected results:
extern crate rand;
const ROWS: usize = 100000;
const COLS: usize = 10000;
fn main() {
// let mut twodarray: Box<[[u8; COLS]; ROWS]> = Box::new([[0; COLS]; ROWS]);
// for i in 0..ROWS {
// for j in 0..COLS {
// twodarray[i][j] = rand::random::<u8>();
// }
// }
// let mut sum: u32 = 0;
// for k in 0..ROWS {
// for l in 0..COLS {
// sum += twodarray[k][l] as u32;
// }
// }
let mut twodarray: Box<[[u8; ROWS]; COLS]> = Box::new([[0; ROWS]; COLS]);
for i in 0..COLS {
for j in 0..ROWS {
twodarray[i][j] = rand::random::<u8>();
}
}
let mut sum: u32 = 0;
for k in 0..COLS {
for l in 0..ROWS {
sum += twodarray[k][l] as u32;
}
}
println!("{}", sum);
}
This two-dimensional array is huge. Rust tries to allocate it on the stack (that's the way arrays in Rust work).
You explicitly wrote you didn't want to use vectors, but in reality vectors are allocated on the heap, so you wouldn't have such a problem with them. It's good to keep that in mind.
If you insist on using arrays, maybe put them in a Box, so that they go to the heap?
That's not a Rust-specific problem. Stacks are generally a lot more limited, compared to the heap, usually up to a few megabytes. In some other languages, such as Java, arrays are allocated on the heap, but that doesn't mean there is no similar stack size limit there as well.
I am trying to make the method with a class created with neon and Rust async. but I m not sure how to do so for a class's method.
For example, I would want to have myMethod as an async.
use neon::prelude::*;
pub struct SomeClass{
path: String,
}
declare_types! {
pub class MyClass for SomeClass {
init(mut cx) {
let path = cx.argument::<JsString>(0)?.value();
Ok(Predictor {
path: path ,
})
}
method myMethod(mut cx) {
let this = cx.this();
let guard = cx.lock();
let test = this.borrow(&guard).path.unwrap();
let vector = vec![1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3,];
// Iterate over the Rust Vec and map each value in the Vec to the JS array
let js_array = JsArray::new(&mut cx, vector.len() as u32);
for (i, obj) in preds.iter().enumerate() {
let element = cx.number(f64::from(*obj));
js_array.set(&mut cx, i as u32, element).unwrap();
}
Ok(js_array.upcast())
}
}
}
// Export the class
register_module!(mut m, {
// <JsEmployee> tells neon what class we are exporting
// "Employee" is the name of the export that the class is exported as
m.export_class::<myClass>("MyClass")?;
Ok(())
});
This is currently a todo in neon. But you can create a simple background task and wrap it in Promise pretty easily. See documentation for examples.
Rust
use neon::prelude::*;
struct BackgroundTask;
impl Task for BackgroundTask {
type Output = i32;
type Error = String;
type JsEvent = JsNumber;
fn perform(&self) -> Result<Self::Output, Self::Error> {
Ok(17)
}
fn complete(self, mut cx: TaskContext, result: Result<Self::Output, Self::Error>) -> JsResult<Self::JsEvent> {
Ok(cx.number(result.unwrap()))
}
}
pub fn perform_async_task(mut cx: FunctionContext) -> JsResult<JsUndefined> {
let f = cx.argument::<JsFunction>(0)?;
BackgroundTask.schedule(f);
Ok(cx.undefined())
}
register_module!(mut m, {
m.export_function("performAsyncTask", perform_async_task)
});
JS
// ./lib/index.js
const { performAsyncTask } = require("../native");
// Iterate 10,0000 times in background thread
const promisePerformAsyncTask = () => {
return new Promise((resolve, reject) => {
performAsyncTask((err, res) => {
let count = 10;
for (let i = 0; i < 100000; i++) {
count++;
}
console.log(count, "first sum from background thread");
});
});
};
I'm new to Rust and I'd like to reduce the nesting of my code. Take this C# code as an example:
for (int i = 0; i < 100; i++)
{
var obj = arr[i];
if (obj != null)
{
var something = obj.Something;
if (something == null)
{
if (i % 3 == 0)
Console.WriteLine(i);
break;
}
}
}
ReSharper suggests rewriting it to reduce nesting:
for (int i = 0; i < 100; i++)
{
var obj = arr[i];
if (obj == null)
continue;
var something = obj.Something;
if (something != null)
continue;
if (i % 3 == 0)
Console.WriteLine(i);
break;
}
Is there way to do the same thing in Rust? If I have nested Option<Option<Option<T>>> and so on I should write something like:
fn main() {
for i in 1..100 {
if let Some(data) = some_data::get_some_data() {
if let Some(result) = data.some_work(i) {
if result > 80 {
break;
}
}
}
println!("{}", i);
}
}
mod some_data
{
pub struct SomeData {
value : i32
}
impl SomeData {
pub fn some_work(&self, i : i32) -> Option<i32> {
Some(self.value + i)
}
}
pub fn get_some_data() -> Option<SomeData> {
Some(SomeData { value : 50 })
}
}
this example is simplified, but it shows the core problem.
There are a few things you could do. One option would be to use a match and continue in the None arm, similar to your C# code:
fn main() {
for i in 1..100 {
let data = match some_data::get_some_data() {
None => continue,
Some(data) => data
};
let result = match data.some_work(i) {
None => continue,
Some(result) => result
};
if result > 80 {
break;
}
println!("{}", i);
}
}
Another thing you could do, as #Cecilio Pardo suggested, is use the is_then method of the Option type to chain your operations:
fn main() {
for i in 1..100 {
if let Some(result) = some_data::get_some_data()
.and_then(|data| data.some_work(i)) {
if result > 80 {
break;
}
}
}
}
I am trying to bundle state in a struct together with callbacks that mutate the state. It works fine when I use managed pointers:
struct StateAndCallbacks01 {
state: #mut int,
inc: #fn(),
dec: #fn()
}
let state01: #mut int = #mut 0;
let inc01: #fn() = || {
*state01 += 1;
};
let dec01: #fn() = || {
*state01 -= 1;
};
let state_cbs_01 = #StateAndCallbacks01 {
state: state01,
inc: inc01,
dec: dec01
};
(state_cbs_01.inc)();
println(fmt!("state: %d", *state_cbs_01.state));
(state_cbs_01.dec)();
println(fmt!("state: %d", *state_cbs_01.state));
Next, I want to send this structure to another task, and thus have to switch to unique pointers everywhere. I cannot make that work: "error: obsolete syntax: const or mutable owned pointer"
struct StateAndCallbacks02 {
state: ~mut int,
inc: ~fn(),
dec: ~fn()
}
let state02: ~mut int = ~mut 0;
let inc02: ~fn() = || {
*state02 += 1;
};
let dec02: ~fn() = || {
*state02 -= 1;
};
let state_cbs_02 = ~StateAndCallbacks02 {
state: state02,
inc: inc02,
dec: dec02
};
let (port, chan): (Port<bool>, Chan<bool>) = stream();
do spawn {
(state_cbs_02.inc)();
println(fmt!("state: %d", *state_cbs_02.state));
(state_cbs_02.dec)();
println(fmt!("state: %d", *state_cbs_02.state));
chan.send(true);
};
let result = port.recv();
println(fmt!("result: %s", result));
Any suggestions? Any better ways to send callbacks across tasks?
Instead of keeping the functions as fields in the struct you can instead add methods to the struct.
struct Foo {
data: int
}
impl Foo {
fn inc(&mut self) {
self.data += 1;
}
}
The impl syntax lets you define methods on the struct. Which you can later call:
let mut my_foo = Foo { data: 0 };
my_foo.inc();
You have to declare my_foo as mutable since the inc method needs to take a mutable reference to it.
The reason for the obsolete syntax error is because doing ~mut 0 is deprecated since mutability is determined by who 'owns' the object. What you'd have to do instead is let mut foo = ~0. The variable foo is the 'owner' and thus is where you declare the mutability. #-pointers are special in that they don't inherit mutability and are managed by a task local GC. (Sections 8 & 9 of the Rust tutorial better explain this)
So with all that put together, you could write your original code like so:
struct State {
data: int
}
impl State {
fn inc(&mut self) {
self.data += 1;
}
fn dec(&mut self) {
self.data -= 1;
}
}
fn main() {
let state = State {
data: 0
};
let (port, chan) = stream();
do spawn {
let mut state = state;
state.inc();
println(fmt!("State: %d", state.data));
state.dec();
println(fmt!("State: %d", state.data));
chan.send(true);
};
let result = port.recv();
println(fmt!("Result: %?", result));
}