pyzmq Segmentation fault on large .recv() - pyzmq

Question

edited - I think I wrongly pointed the blame at pyzmq - I actually was running in a QThread and upon suggestion worked on a full working example - not using QThread and I do not see any issues. I'm going to drop my working example for anyone who may want to reference this example.
I'm attempting to publish data on a pyzmq socket. I have experience here but never for larger data. On occasion, I'm getting a seg fault or at other times a message about free(). This happens more often for larger data sets, up to a point where it never actually works (depending on size).
subscriber code:
import zmq
from time import sleep
from threading import Thread
class ZmqListener(Thread):
def __init__(self, addr, port):
Thread.__init__(self)
self.addr = addr
self.port = port
def run(self):
socket = zmq.Context().socket(zmq.SUB)
socket.connect("tcp://%s:%s" % (self.addr, self.port))
socket.setsockopt(zmq.SUBSCRIBE, "test:".encode('utf-8'))
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
self.running = True
while self.running:
print(socket.recv())
s = dict(poller.poll(1000))
if s:
if s.get(socket) == zmq.POLLIN:
msg = socket.recv()
print(msg)
socket.close()
th = ZmqListener('localhost', IMG_BASE_PORT)
th.start()
while True:
try:
sleep(1)
except KeyboardInterrupt:
break
th.running = False
th.join()
QThread subscriber code that breaks at random:
class ZmqListener(QtCore.QThread):
message = QtCore.Signal(object)
def __init__(self, addr, port, parent=None):
QtCore.QThread.__init__(self, parent)
self.addr = addr
self.port = port
def run(self):
socket = zmq.Context().socket(zmq.SUB)
socket.connect("tcp://%s:%s" % (self.addr, self.port))
socket.setsockopt(zmq.SUBSCRIBE, "{}:".format(IMG_SUBS_HEAD).encode('utf-8'))
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
self.running = True
while self.running:
s = dict(poller.poll(1000))
if s:
if s.get(socket) == zmq.POLLIN:
msg = socket.recv()
self.message.emit(msg[len(IMG_SUBS_HEAD)+1::])
socket.close()
publisher code:
#include <zmq.hpp>
#include <string>
#include <sstream>
#include <cstdlib>
#include <ios>
#include <thread>
#include <chrono>
#define PORT 13600
#define PUBS "test:"
#define NBYT 256 * 1024 * 1024 * 1024
int main()
{
zmq::context_t context(1);
zmq::socket_t socket(context, ZMQ_PUB);
std::string rep_head;
char bytes[NBYT];
std::string zaddr("tcp://*:");
zaddr += std::to_string(PORT);
socket.bind(zaddr.c_str());
rep_head = std::string(PUBS);
int idx = 0;
while (1) {
for (int i = 0; i < NBYT; i++)
bytes[i] = rand(); // sleazy
std::stringstream ss;
ss << rep_head << std::hex << (int)idx << ":";
zmq::message_t msg(ss.str().length() + NBYT);
memcpy((char*)msg.data(), ss.str().data(), ss.str().length());
memcpy(((char*)msg.data())+ss.str().length(), bytes, NBYT);
socket.send(msg);
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
socket.close();
return 0;
}
I've recently updated my OS (Ubuntu 18.04) and this was not a problem in the past.

Answer 1

Step 1: Never violate API rules. Never worth trying to do that
PUB side ought follow the ZeroMQ API documentation: never access messages data "manually", always use { zmq_msg_init() | zmq_msg_init_data() | zmq_msg_init_size() }-mutually-exclusive member-functions, without ever touching the fragile eggs directly ( content referenced by the pointers )
void myFuncToFREE ( void *data2FREE, void *hint ) // UTILITY FUN FOR FREE-ING MEMORY
{ // STRAIGHT ON .SEND()
free ( data2FREE );
}
...
zmq_msg_t message;
int message_len = ss.str().length() + NBYT;
// int zmq_msg_init_data ( zmq_msg_t *msg, void *data, size_t size, zmq_free_fun *ffn, void *hint );
rc = zmq_msg_init_data ( &message, // -------------- LOAD AT ONCE:
< here put concatenated both ss.str().data() + bytes >,
message_len,
myFuncToFREE,
NULL
);
assert ( rc == 0 && "INF: FAILED call to zmq_msg_init_data() ... " );
...
rc = zmq_msg_send( &message, socket, 0 );
assert ( rc == message_len && "INF: FAILED call to zmq_msg_send() ... " );
...
rc = zmq_msg_close( &message ); // Note that this is NOT necessary after a successful zmq_msg_send()
assert ( rc == 0 && "INF: FAILED call to zmq_msg_close() ... " );