I have a simple C-extension(see example below) that sometimes prints using the printf function.
I'm looking for a way to wrap the calls to the function from that C-extensions so that all those printfs will be redirected to my python logger.
hello.c:
#include <Python.h>
static PyObject* hello(PyObject* self)
{
printf("example print from a C code\n");
return Py_BuildValue("");
}
static char helloworld_docs[] =
"helloworld(): Any message you want to put here!!\n";
static PyMethodDef helloworld_funcs[] = {
{"hello", (PyCFunction)hello,
METH_NOARGS, helloworld_docs},
{NULL}
};
static struct PyModuleDef cModPyDem =
{
PyModuleDef_HEAD_INIT,
"helloworld",
"Extension module example!",
-1,
helloworld_funcs
};
PyMODINIT_FUNC PyInit_helloworld(void)
{
return PyModule_Create(&cModPyDem);
};
setup.py:
from distutils.core import setup, Extension
setup(name = 'helloworld', version = '1.0', \
ext_modules = [Extension('helloworld', ['hello.c'])])
to use first run
python3 setup.py install
and then:
import helloworld
helloworld.hello()
I want to be able to do something like this:
with redirect_to_logger(my_logger)
helloworld.hello()
EDIT: I saw a number of posts showing how to silence the prints from C, but I wasn't able to figure out from it how can I capture the prints in python instead.
Example of such post: Redirect stdout from python for C calls
I assume that this question didn't get much traction because I maybe ask too much, so I don't care about logging anymore... how can I capture the C prints in python? to a list or whatever.
EDIT
So I was able to achieve somewhat a working code that does what I want - redirect c printf to python logger:
import select
import threading
import time
import logging
import re
from contextlib import contextmanager
from wurlitzer import pipes
from helloworld import hello
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
class CPrintsHandler(threading.Thread):
def __init__(self, std, poll_std, err, poll_err, logger):
super(CPrintsHandler, self).__init__()
self.std = std
self.poll_std = poll_std
self.err = err
self.poll_err = poll_err
self.logger = logger
self.stop_event = threading.Event()
def stop(self):
self.stop_event.set()
def run(self):
while not self.stop_event.is_set():
# How can I poll both std and err at the same time?
if self.poll_std.poll(1):
line = self.std.readline()
if line:
self.logger.debug(line.strip())
if self.poll_err.poll(1):
line = self.err.readline()
if line:
self.logger.debug(line.strip())
#contextmanager
def redirect_to_logger(some_logger):
handler = None
try:
with pipes() as (std, err):
poll_std = select.poll()
poll_std.register(std, select.POLLIN)
poll_err = select.poll()
poll_err.register(err, select.POLLIN)
handler = CPrintsHandler(std, poll_std, err, poll_err, some_logger)
handler.start()
yield
finally:
if handler:
time.sleep(0.1) # why do I have to sleep here for the foo prints to finish?
handler.stop()
handler.join()
def foo():
logger.debug('logger print from foo()')
hello()
def main():
with redirect_to_logger(logger):
# I don't want the logs from here to be redirected as well, only printf.
logger.debug('logger print from main()')
foo()
main()
But I have a couple of issues:
The python logs are also being redirected and caught by the CPrintsHandler. Is there a way to avoid that?
The prints are not exactly in the correct order:
python3 redirect_c_example_for_stackoverflow.py
2020-08-18 19:50:47,732 - root - DEBUG - example print from a C code
2020-08-18 19:50:47,733 - root - DEBUG - 2020-08-18 19:50:47,731 - root - DEBUG - logger print from main()
2020-08-18 19:50:47,733 - root - DEBUG - 2020-08-18 19:50:47,731 - root - DEBUG - logger print from foo()
Also, the logger prints all go to err, perhaps the way I poll them causes this order.
I'm not that familiar with select in python and not sure if there is a way to poll both std and err at the same time and print whichever has something first.
On Linux you could use wurlitzer which would capture the output from fprint, e.g.:
from wurlitzer import pipes
with pipes() as (out, err):
helloworld.hello()
out.read()
#'example print from a C code\n'
wurlitzer is based on this article of Eli Bendersky, the code from which you can use if you don't like to depend on third-party libraries.
Sadly, wurlitzer and the code from the article work only for Linux (and possible MacOS).
Here is a prototype (an improved version of the prototype can be installed from my github) for Windows using Eli's approach as Cython-extension (which probably could be translated to ctypes if needed):
%%cython
import io
import os
cdef extern from *:
"""
#include <windows.h>
#include <io.h>
#include <stdlib.h>
#include <stdio.h>
#include <fcntl.h>
int open_temp_file() {
TCHAR lpTempPathBuffer[MAX_PATH+1];//path+NULL
// Gets the temp path env string (no guarantee it's a valid path).
DWORD dwRetVal = GetTempPath(MAX_PATH, // length of the buffer
lpTempPathBuffer); // buffer for path
if(dwRetVal > MAX_PATH || (dwRetVal == 0))
{
return -1;
}
// Generates a temporary file name.
TCHAR szTempFileName[MAX_PATH + 1];//path+NULL
DWORD uRetVal = GetTempFileName(lpTempPathBuffer, // directory for tmp files
TEXT("tmp"), // temp file name prefix
0, // create unique name
szTempFileName); // buffer for name
if (uRetVal == 0)
{
return -1;
}
HANDLE tFile = CreateFile((LPTSTR)szTempFileName, // file name
GENERIC_READ | GENERIC_WRITE, // first we write than we read
0, // do not share
NULL, // default security
CREATE_ALWAYS, // overwrite existing
FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE, // "temporary" temporary file, see https://learn.microsoft.com/en-us/archive/blogs/larryosterman/its-only-temporary
NULL); // no template
if (tFile == INVALID_HANDLE_VALUE) {
return -1;
}
return _open_osfhandle((intptr_t)tFile, _O_APPEND | _O_TEXT);
}
int replace_stdout(int temp_fileno)
{
fflush(stdout);
int old;
int cstdout = _fileno(stdout);
old = _dup(cstdout); // "old" now refers to "stdout"
if (old == -1)
{
return -1;
}
if (-1 == _dup2(temp_fileno, cstdout))
{
return -1;
}
return old;
}
int restore_stdout(int old_stdout){
fflush(stdout);
// Restore original stdout
int cstdout = _fileno(stdout);
return _dup2(old_stdout, cstdout);
}
void rewind_fd(int fd) {
_lseek(fd, 0L, SEEK_SET);
}
"""
int open_temp_file()
int replace_stdout(int temp_fileno)
int restore_stdout(int old_stdout)
void rewind_fd(int fd)
void close_fd "_close" (int fd)
cdef class CStdOutCapture():
cdef int tmpfile_fd
cdef int old_stdout_fd
def start(self): #start capturing
self.tmpfile_fd = open_temp_file()
self.old_stdout_fd = replace_stdout(self.tmpfile_fd)
def stop(self): # stops capturing, frees resources and returns the content
restore_stdout(self.old_stdout_fd)
rewind_fd(self.tmpfile_fd) # need to read from the beginning
buffer = io.TextIOWrapper(os.fdopen(self.tmpfile_fd, 'rb'))
result = buffer.read()
close_fd(self.tmpfile_fd)
return result
And now:
b = CStdOutCapture()
b.start()
helloworld.hello()
out = b.stop()
print("HERE WE GO:", out)
# HERE WE GO: example print from a C code
This is what I would do if I am free to edit the C code. Open a memory map in C and write to its file descriptor using fprintf(). Expose the file descriptor to Python either as the int and then use mmap module to open it or use os.openfd() to wrap it in a simpler file-like object, or wrap it in file-like object in C and let Python use that.
Then I would create a class that will enable me to write to sys.stdout through usual interface, i.e. its write() method (for Python's side usage) , and that would use select module to poll the file from C that acts as its stdout in a thread. Then I would switch sys.stdout with an object of this class. So, when Python does sys.stdout.write(...) the string will be redirected to sys.stdout.write(), and when the loop in a thread detects output on a file from C, it will write it using sys.stdout.write(). So, everything will be written to the screen and be available to loggers as well.
In this model, the strictly C part will never actually be writing to the file descriptor connected to the terminal.
You can even do much of this in C itself and leave little for the Python's side, but its easier to influence the interpreter from the Python's side as the extension is the shared library which involves some kind of, lets call it, IPC and OS in the whole story. That's why the stdout is not shared between extension and Python in the first place.
If you want to continue printf() on C side, you can see how you can redirect it in C before programming this whole mess.
This answer is strictly theoretical because I have no time to test it; but it should be doable according to my knowledge. If you try it, please let me know in a comment how it went. Perhaps I missed something, but, I am certain the theory is sane.
Beauty of this idea is that it will be OS independent, although the part with shared memory or connecting a file descriptor to allocated space in RAM can be sometimes PITA on Windows.
If you are not constrained to using the printf in C, it would be easier to use the print equivalent from python C API and pass where you want to redirect the message as an argument.
For example, your hello.c would be:
#include <Python.h>
static PyObject* hello(PyObject* self, PyObject *args)
{
PyObject *file = NULL;
if (!PyArg_ParseTuple(args, "O", &file))
return NULL;
PyObject *pystr = PyUnicode_FromString("example print from a C code\n");
PyFile_WriteObject(pystr, file, Py_PRINT_RAW);
return Py_BuildValue("");
}
static char helloworld_docs[] =
"helloworld(): Any message you want to put here!!\n";
static PyMethodDef helloworld_funcs[] = {
{"hello", (PyCFunction)hello,
METH_VARARGS, helloworld_docs},
{NULL}
};
static struct PyModuleDef cModPyDem =
{
PyModuleDef_HEAD_INIT,
"helloworld",
"Extension module example!",
-1,
helloworld_funcs
};
PyMODINIT_FUNC PyInit_helloworld(void)
{
return PyModule_Create(&cModPyDem);
};
We can check if it is working with the program below:
import sys
import helloworld
helloworld.hello(sys.stdout)
helloworld.hello(sys.stdout)
helloworld.hello(sys.stderr)
In the command line we redirect each output separately:
python3 example.py 1> out.txt 2> err.txt
out.txt will have two print calls, while err.txt will have only one, as expected from our python script.
You can check python's print implementation to get some more ideas of what you can do.
cpython print source code
I am trying to get idle time from xlib screensaver
i tried debugging it, and the error starts happening after the line
dpy = xlib.XOpenDisplay(os.environ['DISPLAY'].encode('ascii'))
Here's my code
class XScreenSaverInfo( ctypes.Structure):
""" typedef struct { ... } XScreenSaverInfo; """
_fields_ = [('window', ctypes.c_ulong), # screen saver window
('state', ctypes.c_int), # off,on,disabled
('kind', ctypes.c_int), # blanked,internal,external
('since', ctypes.c_ulong), # milliseconds
('idle', ctypes.c_ulong), # milliseconds
('event_mask', ctypes.c_ulong)] # events
xlib = ctypes.cdll.LoadLibrary('libX11.so')
dpy = xlib.XOpenDisplay(os.environ['DISPLAY'].encode('ascii'))
root = xlib.XDefaultRootWindow(dpy)
xss = ctypes.cdll.LoadLibrary( 'libXss.so')
xss.XScreenSaverAllocInfo.restype = ctypes.POINTER(XScreenSaverInfo)
xss_info = xss.XScreenSaverAllocInfo()
xss.XScreenSaverQueryInfo( dpy, root, xss_info)
print("Idle time in milliseconds: %d") % xss_info.contents.idle
I get a Segmentation fault (core dumped) error.
Please help:)
The error is because argtypes (and restype) not being specified, as described in [Python 3.Docs]: ctypes - Specifying the required argument types (function prototypes). There are many examples of what happens in this case, here are 2 of them:
[SO]: python ctypes issue on different OSes (#CristiFati's answer)
[SO]: Python ctypes cdll.LoadLibrary, instantiate an object, execute its method, private variable address truncated (#CristiFati's answer)
Declare them for each function, before calling it:
xlib.XOpenDisplay.argtypes = [ctypes.c_char_p]
xlib.XOpenDisplay.restype = ctypes.c_void_p # Actually, it's a Display pointer, but since the Display structure definition is not known (nor do we care about it), make it a void pointer
xlib.XDefaultRootWindow.argtypes = [ctypes.c_void_p]
xlib.XDefaultRootWindow.restype = ctypes.c_uint32
xss.XScreenSaverQueryInfo.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.POINTER(XScreenSaverInfo)]
xss.XScreenSaverQueryInfo.restype = ctypes.c_int
I am trying to use Collections-C in Cython.
I noticed that some structures are defined in the .c file, and an alias for them is in the .h file. When I try to define those structures in a .pxd file and use them in a .pyx file, gcc throws an error: storage size of ‘[...]’ isn’t known.
I was able to reproduce my issue to a minimum setup that replicates the external library and my application:
testdef.c
/* Note: I can't change this */
struct bogus_s {
int x;
int y;
};
testdef.h
/* Note: I can't change this */
typedef struct bogus_s Bogus;
cytestdef.pxd
# This is my code
cdef extern from 'testdef.h':
struct bogus_s:
int x
int y
ctypedef bogus_s Bogus
cytestdef.pyx
# This is my code
def fn():
cdef Bogus n
n.x = 12
n.y = 23
print(n.x)
If I run cythonize, I get
In function ‘__pyx_pf_7sandbox_9cytestdef_fn’:
cytestdef.c:1106:9: error: storage size of ‘__pyx_v_n’ isn’t known
Bogus __pyx_v_n;
^~~~~~~~~
I also get the same error if I use ctypedef Bogus: [...] notation as indicated in the Cython manual.
What am I doing wrong?
Thanks.
Looking at the documentation for your Collections-C library these are opaque structures that you're supposed to use purely through pointers (don't need to know the size to have a pointer, while you do to allocate on the stack). Allocation of these structures is done in library functions.
To change your example to match this case:
// C file
int bogus_s_new(struct bogus_s** v) {
*v = malloc(sizeof(struct bogus_s));
return (v!=NULL);
}
void free_bogus_s(struct bogus_s* v) {
free(v);
}
Your H file would contain the declarations for those and your pxd file would contain wrappers for the declarations. Then in Cython:
def fn():
cdef Bogus* n
if not bogus_s_new(&n):
return
try:
# you CANNOT access x and y since the type is
# designed to be opaque. Instead you should use
# the acessor functions defined in the header
# n.x = 12
# n.y = 23
finally:
free_bogus_s(n)
I am struggling to initialise thread-local ndarrays with cython.parallel:
Pseudo-code:
cdef:
ndarray buffer
with nogil, parallel():
buffer = np.empty(...)
for i in prange(n):
with gil:
print "Thread %d: data address: 0x%x" % (threadid(), <uintptr_t>buffer.data)
some_func(buffer.data) # use thread-local buffer
cdef void some_func(char * buffer_ptr) nogil:
(... works on buffer contents...)
My problem is that in all threads buffer.data points to the same address. Namely the address of the thread that last assigned buffer.
Despite buffer being assigned within the parallel() (or alternatively prange) block, cython does not make buffer a private or thread-local variable but keeps it as a shared variable.
As a result, buffer.data points to the same memory region wreaking havoc on my algorithm.
This is not a problem exclusively with ndarray objects but seemingly with all cdef class defined objects.
How do I solve this problem?
I think I have finally found a solution to this problem that I like.
The short version is that you create an array that has shape:
(number_of_threads, ...<whatever shape you need in the thread>...)
Then, call openmp.omp_get_thread_num and use that to index the array to get a "thread-local" sub-array. This avoids having a separate array for every loop index (which could be enormous) but also prevents threads overwriting each other.
Here's a rough version of what I did:
import numpy as np
import multiprocessing
from cython.parallel cimport parallel
from cython.parallel import prange
cimport openmp
cdef extern from "stdlib.h":
void free(void* ptr)
void* malloc(size_t size)
void* realloc(void* ptr, size_t size)
...
cdef int num_items = ...
num_threads = multiprocessing.cpu_count()
result_array = np.zeros((num_threads, num_items), dtype=DTYPE) # Make sure each thread uses separate memory
cdef c_numpy.ndarray result_cn
cdef CDTYPE ** result_pointer_arr
result_pointer_arr = <CDTYPE **> malloc(num_threads * sizeof(CDTYPE *))
for i in range(num_threads):
result_cn = result_array[i]
result_pointer_arr[i] = <CDTYPE*> result_cn.data
cdef int thread_number
for i in prange(num_items, nogil=True, chunksize=1, num_threads=num_threads, schedule='static'):
thread_number = openmp.omp_get_thread_num()
some_function(result_pointer_arr[thread_number])
I'm wrapping a C library to Pyhon 3 using Cython and i'm looking for a way of converting wchar_t string to python object which i want to return from a function. There's an answer in this question, but it involves encoding the string as multibyte str, and decoding it back to unicode. I hope for more straightforward solution. I tried using PyUnicode_FromWideChar from Python C API, but i'm getting a segfault. Here's my .pyx code:
from cpython.ref cimport PyObject
from libc.stddef cimport wchar_t
cdef extern from "Python.h":
PyObject* PyUnicode_FromWideChar(wchar_t *w, Py_ssize_t size)
cdef extern from "../../src/my_c_lib.h":
wchar_t * myObjToStr(wchar_t * result, size_t size, myObj * obj)
cdef class MyClass:
cdef myObj * ptr
...
def to_str(self):
cdef wchar_t buf[64]
cdef wchar_t * result = myObjToStr(buf, 64, self.ptr)
if result is NULL:
raise Exception("Error converting object to string.")
cdef PyObject * pystr = PyUnicode_FromWideChar(result, 64)
return <object>pystr
result is actually the pointer to buf. What's wrong with this? Is there another way without encoding/decoding?
Edit: I found that PyUnicode_FromWideChar() returns NULL, but why? I checked, that result is a valid wchar_t * string.
Using -1 as second argument to PyUnicode_FromWideChar(result, -1)(so that the function uses wcslen internally), fixed the problem.
So PyUnicode_FromWideChar really is the way to go.