I'm trying to develop a program which runs continuously.
It should pull some data from a database every sleepPool seconds and 'process' the information in a non-blocking way(at least that's what I'm trying to do).
The issue is that the memory keeps growing so I'm wondering if I'm doing something wrong.
Below is a snippet from the my program.
var uCh = make(chan *user, buffLimit) //emits new users to process
var statsCh = make(chan *user, buffLimit) //emits new users to store
func main() {
go emitUser(db)
go consumeUser(db)
for ur := range statsCh {
log.Infoln(ur)
}
}
func emitUser(db *sql.DB) {
for {
time.Sleep(sleepPool * time.Second)
log.Infoln("looking for new users")
rows, err := rowsD.Query()
for rows.Next() {
uCh <- usr
}
}
}
func consumeUser(db *sql.DB) {
for usr := range uCh {
go func(usr *user) {
//do something with the user
statsCh <- usr
}(usr)
}
}
I've read that I may need to close the channels so that the gc can recycle the memory but I'm not sure how to do that (because the program should run continuously) and if I really need to do it because the data is always read (guaranteed by the range from main) so I assume the memory is recycled.
You didn't give enough time for the GC to kick in, wait for an hour then check the memory.
If you really really wan (bad idea and gonna slow your program) to force it to free the memory you can use something like:
import "runtime/debug"
//........
func forceFree() {
for _ = range time.Tick(30 * time.Second) {
debug.FreeOSMemory()
}
}
func init() {
go forceFree()
}
This code does a lot of harmless passing things around, but should not leak anything.
My guess is that the rowsD.Query method has some kind of cache, or other leak.
Of course, it could just be fragmentation, or an artefact of the garbage collector (in which case you should see memory use level off, or even drop, over time).
Related
I'm working in a Node environment on Windows. My code is receiving 30 Buffer objects (~500-900kb each) each second, and I need to save this data to the file system as quickly as possible, without engaging in any work that blocks the receipt of the following Buffer (i.e. the goal is to save the data from every buffer, for ~30-45 minutes). For what it's worth, the data is sequential depth frames from a Kinect sensor.
My question is: What is the most performant way to write files in Node?
Here's pseudocode:
let num = 0
async function writeFile(filename, data) {
fs.writeFileSync(filename, data)
}
// This fires 30 times/sec and runs for 30-45 min
dataSender.on('gotData', function(data){
let filename = 'file-' + num++
// Do anything with data here to optimize write?
writeFile(filename, data)
}
fs.writeFileSync seems much faster than fs.writeFile, which is why I'm using that above. But are there any other ways to operate on the data or write to file that could speed up each save?
First off, you never want to use fs.writefileSync() in handling real-time requests because that blocks the entire node.js event loop until the file write is done.
OK, based on writing each block of data to a different file, then you want to allow multiple disk writes to be in process at the same time, but not unlimited disk writes. So, it's still appropriate to use a queue, but this time the queue doesn't just have one write in process at a time, it has some number of writes in process at the same time:
const EventEmitter = require('events');
class Queue extends EventEmitter {
constructor(basePath, baseIndex, concurrent = 5) {
this.q = [];
this.paused = false;
this.inFlightCntr = 0;
this.fileCntr = baseIndex;
this.maxConcurrent = concurrent;
}
// add item to the queue and write (if not already writing)
add(data) {
this.q.push(data);
write();
}
// write next block from the queue (if not already writing)
write() {
while (!paused && this.q.length && this.inFlightCntr < this.maxConcurrent) {
this.inFlightCntr++;
let buf = this.q.shift();
try {
fs.writeFile(basePath + this.fileCntr++, buf, err => {
this.inFlightCntr--;
if (err) {
this.err(err);
} else {
// write more data
this.write();
}
});
} catch(e) {
this.err(e);
}
}
}
err(e) {
this.pause();
this.emit('error', e)
}
pause() {
this.paused = true;
}
resume() {
this.paused = false;
this.write();
}
}
let q = new Queue("file-", 0, 5);
// This fires 30 times/sec and runs for 30-45 min
dataSender.on('gotData', function(data){
q.add(data);
}
q.on('error', function(e) {
// go some sort of write error here
console.log(e);
});
Things to consider:
Experiment with the concurrent value you pass to the Queue constructor. Start with a value of 5. Then see if raising that value any higher gives you better or worse performance. The node.js file I/O subsystem uses a thread pool to implement asynchronous disk writes so there is a max number of concurrent writes that will allow so cranking the concurrent number up really high probably does not make things go faster.
You can experiement with increasing the size of the disk I/O thread pool by setting the UV_THREADPOOL_SIZE environment variable before you start your node.js app.
Your biggest friend here is disk write speed. So, make sure you have a fast disk with a good disk controller. A fast SSD on a fast bus would be best.
If you can spread the writes out across multiple actual physical disks, that will likely also increase write throughput (more disk heads at work).
This is a prior answer based on the initial interpretation of the question (before editing that changed it).
Since it appears you need to do your disk writes in order (all to the same file), then I'd suggest that you either use a write stream and let the stream object serialize and cache the data for you or you can create a queue yourself like this:
const EventEmitter = require('events');
class Queue extends EventEmitter {
// takes an already opened file handle
constructor(fileHandle) {
this.f = fileHandle;
this.q = [];
this.nowWriting = false;
this.paused = false;
}
// add item to the queue and write (if not already writing)
add(data) {
this.q.push(data);
write();
}
// write next block from the queue (if not already writing)
write() {
if (!nowWriting && !paused && this.q.length) {
this.nowWriting = true;
let buf = this.q.shift();
fs.write(this.f, buf, (err, bytesWritten) => {
this.nowWriting = false;
if (err) {
this.pause();
this.emit('error', err);
} else {
// write next block
this.write();
}
});
}
}
pause() {
this.paused = true;
}
resume() {
this.paused = false;
this.write();
}
}
// pass an already opened file handle
let q = new Queue(fileHandle);
// This fires 30 times/sec and runs for 30-45 min
dataSender.on('gotData', function(data){
q.add(data);
}
q.on('error', function(err) {
// got disk write error here
});
You could use a writeStream instead of this custom Queue class, but the problem with that is that the writeStream may fill up and then you'd have to have a separate buffer as a place to put the data anyway. Using your own custom queue like above takes care of both issues at once.
Other Scalability/Performance Comments
Because you appear to be writing the data serially to the same file, your disk writing won't benefit from clustering or running multiple operations in parallel because they basically have to be serialized.
If your node.js server has other things to do besides just doing these writes, there might be a slight advantage (would have to be verified with testing) to creating a second node.js process and doing all the disk writing in that other process. Your main node.js process would receive the data and then pass it to the child process that would maintain the queue and do the writing.
Another thing you could experiment with is coalescing writes. When you have more than one item in the queue, you could combine them together into a single write. If the writes are already sizable, this probably doesn't make much difference, but if the writes were small this could make a big difference (combining lots of small disk writes into one larger write is usually more efficient).
Your biggest friend here is disk write speed. So, make sure you have a fast disk with a good disk controller. A fast SSD would be best.
I have written a service that does this extensively and the best thing you can do is to pipe the input data directly to the file (if you have an input stream as well).
A simple example where you download a file in such a way:
const http = require('http')
const ostream = fs.createWriteStream('./output')
http.get('http://nodejs.org/dist/index.json', (res) => {
res.pipe(ostream)
})
.on('error', (e) => {
console.error(`Got error: ${e.message}`);
})
So in this example there is no intermediate copying involved of the whole file. As the file is read in chunks from the remote http server it is written to the file on disk. This is much more efficient that downloading a whole file from the server, saving that in memory and then writing it to a file on disk.
Streams are a basis of many operations in Node.js so you should study those as well.
One other thing that you should investigate depending on your scenarios is UV_THREADPOOL_SIZE as I/O operations use libuv thread pool that is by default set to 4 and you might fill that up if you do a lot of writing.
There are plenty of examples of how to use WaitGroup to wait for all of a group of goroutines to finish, but what if you want to wait for any one of them to finish without using a semaphore system where some process must be waiting? For example, a producer/consumer scenario where multiple producer threads add multiple entries to a data structure while a consumer is removing them one at a time. In this scenario:
We can't just use the standard producer/consumer semaphore system, because production:consumption is not 1:1, and also because the data structure acts as a cache, so the producers can be "free-running" instead of blocking until a consumer is "ready" to consume their product.
The data structure may be emptied by the consumer, in which case, the consumer wants to wait until any one of the producers finishes (meaning that there might be new things in the data structure)
Question: Is there a standard way to do that?
I've only been able to devise two methods of doing this. Both by using channels as semaphores:
var unitary_channel chan int = make(chan int, 1)
func my_goroutine() {
// Produce, produce, produce!!!
unitary_channel<-0 // Try to push a value to the channel
<-unitary_channel // Remove it, in case nobody was waiting
}
func main() {
go my_goroutine()
go my_goroutine()
go my_goroutine()
for len(stuff_to_consume) { /* Consume, consume, consume */ }
// Ran out of stuff to consume
<-unitary_channel
unitary_channel<-0 // To unblock the goroutine which was exiting
// Consume more
}
Now, this simplistic example has some glaring (but solvable issues), like the fact that main() can't exit if there wasn't at least one go_routine() still running.
The second method, instead of requiring producers to remove the value they just pushed to the channel, uses select to allow the producers to exit when the channel would block them.
var empty_channel chan int = make(chan int)
func my_goroutine() {
// Produce, produce, produce!!!
select {
case empty_channel <- 0: // Push if you can
default: // Or don't if you can't
}
}
func main() {
go my_goroutine()
go my_goroutine()
go my_goroutine()
for len(stuff_to_consume) { /* Consume, consume, consume */ }
// Ran out of stuff to consume
<-unitary_channel
// Consume more
}
Of course, this one will also block main() forever if all of the goroutines have already terminated. So, if the answer to the first question is "No, there's no standard solution to this other than the ones you've come up with", is there a compelling reason why one of these should be used instead of the other?
you could use a channel with a buffer like this
// create a channel with a buffer of 1
var Items = make(chan int, 1)
var MyArray []int
func main() {
go addItems()
go addItems()
go addItems()
go sendToChannel()
for true {
fmt.Println(<- Items)
}
}
// push a number to the array
func addItems() {
for x := 0; x < 10; x++ {
MyArray = append(MyArray, x)
}
}
// push to Items and pop the array
func sendToChannel() {
for true {
for len(MyArray) > 0 {
Items <- MyArray[0]
MyArray = MyArray[1:]
}
time.Sleep(10 * time.Second)
}
}
the for loop in main will loop for ever and print anything that gets added to the channel and the sendToChannel function will block when the array is empty,
this way a producer will never be blocked and a consumer can consume when there are one or more items available
I need to download files, chunk by chunk in multiple threads.
For example, I have 1k files, each file ~100Mb-1Gb and I can download these files only by chunks 4096Kb(each http get request gives me only 4kb).
It might be to long to download it in one thread, so I want to download them, let's say in 20 threads(one thread for one file) and I also need to download a few chunks in each of these threads, simultaneously.
Is there any example that shows such logic?
This is an example of how to set up a concurrent downloader. Things to be aware of are bandwidth, memory, and disk space. You can kill your bandwidth by trying to do to much at once, the same goes for memory. Your downloading pretty big files so memory can be an issue. Another thing to note is that by using gorountines you are losing request order. So if the order of the returned bytes matter, then this will not work because you will have to know the byte order to assemble the file in the end, which would mean that a downloading one at a time is best, unless you implement a way to keep track of the order (maybe some kind of global map[order int][]bytes with mutex to prevent race conditions). An alternative that doesn't involve Go (assuming you have a unix machine for ease) is to use Curl see here http://osxdaily.com/2014/02/13/download-with-curl/
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"sync"
)
// now your going to have to be careful because you can potentially run out of memory downloading to many files at once..
// however here is an example that can be modded
func downloader(wg *sync.WaitGroup, sema chan struct{}, fileNum int, URL string) {
sema <- struct{}{}
defer func() {
<-sema
wg.Done()
}()
client := &http.Client{Timeout: 10}
res, err := client.Get(URL)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
var buf bytes.Buffer
// I'm copying to a buffer before writing it to file
// I could also just use IO copy to write it to the file
// directly and save memory by dumping to the disk directly.
io.Copy(&buf, res.Body)
// write the bytes to file
ioutil.WriteFile(fmt.Sprintf("file%d.txt", fileNum), buf.Bytes(), 0644)
return
}
func main() {
links := []string{
"url1",
"url2", // etc...
}
var wg sync.WaitGroup
// limit to four downloads at a time, this is called a semaphore
limiter := make(chan struct{}, 4)
for i, link := range links {
wg.Add(1)
go downloader(&wg, limiter, i, link)
}
wg.Wait()
}
You can check the implementation in aws-go-sdk.
https://github.com/aws/aws-sdk-go-v2/blob/main/feature/s3/manager/download.go
create n concurrent go routine
download the chunks in the n go routine
use writer.WriteAt() to seek and write to certain position.
I've been struggling to find a memory leak in our application and have been using the pprof tool to understand what's going on.
When I look at the heap, I constantly see the following function and I don't understand why (or if) it's actually a problem.
func CreateClients(raw []byte) bool {
macs := []string{}
conn := FormatConn(raw)
if conn.Ap_Mac != "" {
var wg sync.WaitGroup
var array []Client
c1 := make(chan Client)
clients := FormatClients(conn)
wg.Add(len(clients))
for _, c := range clients {
go func(d Client) {
defer wg.Done()
c1 <- UpdateClients(d)
}(c)
}
go func() {
defer wg.Done()
for {
select {
case client := <-c1:
array = append(array, client)
macs = append(macs, client.Client_Mac)
}
}
}()
wg.Wait()
// Do some other stuff
...
}
The UpdateClients function updates the client model in Mongo. When it returns, I need each client - so I can index it with ES plus I need an array of the macs to do some other stuff.
I've gone through the online examples and thought this was the recommended way to loop through a channel.
My pprof heap looks like this, and grows steadily over a few days:
7.53MB of 9.53MB total (79.00%)
Dropped 234 nodes (cum <= 0.05MB)
Showing top 5 nodes out of 28 (cum >= 1MB)
flat flat% sum% cum cum%
2MB 21.00% 21.00% 2MB 21.00% strings.Replace
1.51MB 15.89% 36.89% 1.51MB 15.89% github.com/PolkaSpots/worker/worker.funcĀ·006
1.51MB 15.87% 52.76% 1.51MB 15.87% github.com/PolkaSpots/worker/worker.funcĀ·008
1.50MB 15.75% 68.51% 1.50MB 15.75% newproc_m
1MB 10.50% 79.00% 1MB 10.50% gopkg.in/mgo.v2/bson.(*decoder).readStr
Is there a more efficient / recommended way to achieve this?
EDIT_
As suggested, I've altered the loop as so
done := make(chan bool)
go func() {
for {
select {
case client := <-c1:
array = append(array, client)
macs = append(macs, client.Client_Mac)
case <-done:
return
}
}
}()
wg.Wait()
close(done)
The receive loop never breaks:
for {
select {
case client := <-c1:
...
}
It has no stop condition, no timeout, nothing - so it will just hang there forever - even if your whole function exits. and it will leak both the goroutine and the channel.
On top of that, you're deferring a wg.Done when this loop exits, but you're not doing wg.Add to match it. So if this loop ever exits, you will panic.
What you need to do is find some way to stop the for/select loop. Simplest way IMO - add a second channel that receives data after wg.Wait(), but do not do wg.Done() in that goroutine.
I have a Go program that runs continuously and relies entirely on goroutines + 1 manager thread. The main thread simply calls goroutines and otherwise sleeps.
There is a memory leak. The program uses more and more memory until it drains all 16GB RAM + 32GB SWAP and then each goroutine panics. It is actually OS memory that causes the panic, usually the panic is fork/exec ./anotherapp: cannot allocate memory when I try to execute anotherapp.
When this happens all of the worker threads will panic and be recovered and restarted. So each goroutine will panic, be recovered and restarted... at which point the memory usage will not decrease, it remains at 48GB even though there is now virtually nothing allocated. This means all goroutines will always panic as there is never enough memory, until the entire executable is killed and restarted completely.
The entire thing is about 50,000 lines, but the actual problematic area is as follows:
type queue struct {
identifier string
type bool
}
func main() {
// Set number of gorountines that can be run
var xthreads int32 = 10
var usedthreads int32
runtime.GOMAXPROCS(14)
ready := make(chan *queue, 5)
// Start the manager goroutine, which prepared identifiers in the background ready for processing, always with 5 waiting to go
go manager(ready)
// Start creating goroutines to process as they are ready
for obj := range ready { // loops through "ready" channel and waits when there is nothing
// This section uses atomic instead of a blocking channel in an earlier attempt to stop the memory leak, but it didn't work
for atomic.LoadInt32(&usedthreads) >= xthreads {
time.Sleep(time.Second)
}
debug.FreeOSMemory() // Try to clean up the memory, also did not stop the leak
atomic.AddInt32(&usedthreads, 1) // Mark goroutine as started
// Unleak obj, probably unnecessary, but just to be safe
copy := new(queue)
copy.identifier = unleak.String(obj.identifier) // unleak is a 3rd party package that makes a copy of the string
copy.type = obj.type
go runit(copy, &usedthreads) // Start the processing thread
}
fmt.Println(`END`) // This should never happen as the channels are never closed
}
func manager(ready chan *queue) {
// This thread communicates with another server and fills the "ready" channel
}
// This is the goroutine
func runit(obj *queue, threadcount *int32) {
defer func() {
if r := recover(); r != nil {
// Panicked
erstring := fmt.Sprint(r)
reportFatal(obj.identifier, erstring)
} else {
// Completed successfully
reportDone(obj.identifier)
}
atomic.AddInt32(threadcount, -1) // Mark goroutine as finished
}()
do(obj) // This function does the actual processing
}
As far as I can see, when the do function (last line) ends, either by having finished or having panicked, the runit function then ends, which ends the goroutine entirely, which means all of the memory from that goroutine should now be free. This is now what happens. What happens is that this app just uses more and more and more memory until it becomes unable to function, all the runit goroutines panic, and yet the memory does not decrease.
Profiling does not reveal anything suspicious. The leak appears to be outside of the profiler's scope.
Please consider inverting the pattern, see here or below....
package main
import (
"log"
"math/rand"
"sync"
"time"
)
// I do work
func worker(id int, work chan int) {
for i := range work {
// Work simulation
log.Printf("Worker %d, sleeping for %d seconds\n", id, i)
time.Sleep(time.Duration(rand.Intn(i)) * time.Second)
}
}
// Return some fake work
func getWork() int {
return rand.Intn(2) + 1
}
func main() {
wg := new(sync.WaitGroup)
work := make(chan int)
// run 10 workers
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
worker(i, work)
wg.Done()
}(i)
}
// main "thread"
for i := 0; i < 100; i++ {
work <- getWork()
}
// signal there is no more work to be done
close(work)
// Wait for the workers to exit
wg.Wait()
}