Related
I am writing some software to handle serial port read/writing for a Beaglebone system. The OS is Debian 9. I am writing code in C with --std=gnu99.
Here is my code:
// reference
// https://www.cmrr.umn.edu/~strupp/serial.html
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <termios.h>
#include <sys/ioctl.h>
int open_port(void)
{
int fd;
fd = open("/dev/ttyS1", O_RDWR | O_NOCTTY | O_NDELAY);
// removing O_NDELAY no difference
if(fd == -1)
{
perror("open_port: Unable to open /dev/ttyS1 - ");
}
else
{
fcntl(fd, F_SETFL, 0);
}
return fd;
}
int main()
{
// open fd for serial port
int fd = open_port();
// set baud rate
struct termios options;
// get current options
tcgetattr(fd, &options);
// set input and output baud
cfsetispeed(&options, B115200);
cfsetospeed(&options, B115200);
// enable reciever and set local mode
// CLOCAL: ignore modem control lines
// CREAD: enable reciever
options.c_cflag |= (CLOCAL | CREAD);
// set partity bit
//options.c_cflag &= PARENB;
options.c_cflag &= ~PARENB;
// use even parity
//options.c_cflag &= ~PARODD;
// use only 1 stop bit
options.c_cflag &= ~CSTOPB;
// set character size to 8 bit
options.c_cflag &= ~CSIZE;
options.c_cflag &= CS8;
// disable flow control
//options.c_cflag &= ~CNEW_RTSCTS; // does not work?
// note: check local options, some may be required
// disable canonical input (use raw)
// disable echoing of characters
// disable echoing of erase characters
// disable signals SIGINTR SIGSUSP SIGDSUSP, SIGQUIT
// input characters are passed through exactly as recieved
options.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG);
// disable parity check
options.c_iflag &= IGNPAR;
// disable flow control (software)
options.c_iflag &= ~(IXON | IXOFF | IXANY);
// set raw output, no output processing
options.c_oflag &= ~OPOST;
// set options
tcsetattr(fd, TCSANOW, &options);
char data[] = {0x01, 0x03, 0x00, 0x00};
int n = write(fd, data, 4);
if(n =! 4)
{
printf("write fail %d\n", n);
}
char buffer[40];
n = read(fd, buffer, 2);
if(n != 2)
{
printf("read fail %d\n", n);
}
if(buffer[0] == 0x03 && buffer[1] == 0x00)
{
}
else
{
printf("uart error\n");
}
char data2[] = {0x00, 0x00, 0x00, 0x01};
n = write(fd, data2, 4);
if(n != 4)
{
printf("write fail %d\n", n);
}
n = read(fd, buffer, 2);
if(n != 2)
{
printf("read fail %d\n", n);
}
if(buffer[0] == 0x03 && buffer[1] == 0x00)
{
}
else
{
printf("uart error\n");
}
printf("process complete\n");
// to close
close(fd);
}
The issue I have is calls to read() do not block. This is not the desired behaviour. These calls should block until 2 bytes of data are received.
My guess would be I have misconfigured an option somewhere. However I don't know where the mistake is, and from what I have researched, this should be reading in blocking mode. (fcntl(fd, F_SETFL, 0);)
The issue I have is calls to read() do not block.
That's actually a conclusion, rather than an observation.
Presumably your program is reporting zero bytes read, which technically is not an error.
... from what I have researched, this should be reading in blocking mode. (fcntl(fd, F_SETFL, 0);)
Correct, you have the file descriptor configured for blocking mode.
My guess would be I have misconfigured an option somewhere.
Yes, your termios configuration is incomplete. (It uses proper bitwise operations, but unfortunately is incomplete and has errors, see ADDENDUM.)
When configured for non-canonical mode, you must also specify the VMIN and VTIME parameters.
When left unspecified (as in your code), the default values of VMIN = 0 and VTIME = 0 are likely to be in effect, which is equivalent to a non-blocking read.
Note that the byte length in the read() syscall has minimal effect on completion behavior (other than capping the number of bytes to be returned).
From the man page for read(2):
read() attempts to read up to count bytes from file descriptor fd into the buffer starting at buf.
Therefore the count parameter is effectively a maximum.
The actual number of bytes read can be between zero and count depending on the values configured for VMIN and VTIME (and the actual data available).
See Linux Blocking vs. non Blocking Serial Read for more details.
To read at least two bytes per syscall (with no time limit), include the following in the termios configuration:
options.c_cc[VMIN] = 2;
options.c_cc[VTIME] = 0;
The above two statements inserted into your program before the tcsetattr() syscall would cause your program to block forever until exactly two bytes can be returned in each read(fd, buffer, 2) statement.
A read(fd, buffer, 4) statement would also block forever, and return either 2, 3, or 4 bytes of data (depending on program timing with respect to data reception & buffering).
ADDENDUM
Turns out there are a few bugs in your termios initialization.
As posted your program neither transmits or receives any data correctly, and you failed to detect these errors or neglect to mention them.
The following statement in your program obliterates all the previous c_cflag operations, and unintentionally reconfigures the baudrate to B0 and the character size to CS5:
options.c_cflag &= CS8;
The correct syntax is:
options.c_cflag |= CS8;
The statement
options.c_lflag &= ~(ICANON | ECHO | ECHOE | ISIG);
should be expanded with more attributes to match cfmakeraw():
options.c_lflag &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN);
The following statement in your program disables every attribute in c_iflag except IGNPAR, and leaves the IGNPAR attribute in an unknown/ambiguous state:
// disable parity check
options.c_iflag &= IGNPAR;
To match cfmakeraw(), it should be changed to:
options.c_iflag &= ~(IGNBRK | BRKINT | PARMRK | ISTRIP
| INLCR | IGNCR | ICRNL);
With all the corrections mentioned, I can get your program to execute as expected.
I am trying to read data sent from the tty/USB0 and print it out with byte format.
Question:
I expect to print out the data once the reading bytes reach 40 However, the time takes much longer than I expect. The read() system call hangs and I believe the data should already be larger than 40. The data will finally be printed out but it should not take so long. Did I make anything wrong in this programming ?
thanks
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <termios.h>
#include <stdio.h>
#define BAUDRATE B9600
#define MODEMDEVICE "/dev/ttyUSB0"
#define FALSE 0
#define TRUE 1
main()
{
int fd,c, res;
struct termios oldtio,newtio;
unsigned char buf[40];
fd = open(MODEMDEVICE, O_RDWR | O_NOCTTY );
if (fd <0) {perror(MODEMDEVICE); exit(-1); }
tcgetattr(fd,&oldtio);
bzero(&newtio, sizeof(newtio));
newtio.c_cflag = BAUDRATE | CS8 | CLOCAL | CREAD;
newtio.c_iflag = IGNPAR | ICRNL;
newtio.c_oflag = 1;
newtio.c_lflag = ICANON;
tcflush(fd, TCIOFLUSH);
tcsetattr(fd,TCSANOW,&newtio);
int i;
while (1) {
res = read(fd,buf,40);
if(res==40){
printf("res reaches 40 \n");
}
printf("res: %d\n",res);
for(i=0;i<res;++i){
printf("%02x ", buf[i]);
}
return;
}
}
--------------------raw mode code------------------------
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <termios.h>
#include <stdio.h>
#define BAUDRATE B9600
#define MODEMDEVICE "/dev/ttyUSB0"
#define _POSIX_SOURCE 1 /* POSIX compliant source */
#define FALSE 0
#define TRUE 1
volatile int STOP=FALSE;
main()
{
int fd,c, res;
struct termios oldtio,newtio;
unsigned char buf[255];
fd = open(MODEMDEVICE, O_RDWR | O_NOCTTY );
if (fd <0) {perror(MODEMDEVICE); exit(-1); }
tcgetattr(fd,&oldtio); /* save current port settings */
bzero(&newtio, sizeof(newtio));
newtio.c_cflag = BAUDRATE | CRTSCTS | CS8 | CLOCAL | CREAD;
newtio.c_iflag = IGNPAR;
newtio.c_oflag = 0;
/* set input mode (non-canonical, no echo,...) */
newtio.c_lflag = 0;
newtio.c_cc[VTIME] = 0;
newtio.c_cc[VMIN] = 40;
tcflush(fd, TCIFLUSH);
tcsetattr(fd,TCSANOW,&newtio);
int i;
while (STOP==FALSE) {
res = read(fd,buf,255);
for( i=0;i<res;++i){
printf("%02x \n", buf[i]);
}
}
tcsetattr(fd,TCSANOW,&oldtio);
}
It now can print out the data once buffer capacity is full ( which is 40 ).
1 question:
When I modified the printf
printf("%02x ", buf[i]); ( remove "\n" )
It will not print out when the buffer is full until more bytes are received. Why this happens?
Thanks
You need to switch the terminal to raw mode to disable line buffering.
Citing this answer:
The terms raw and cooked only apply to terminal drivers. "Cooked" is
called canonical and "raw" is called non-canonical mode.
The terminal driver is, by default a line-based system: characters are
buffered internally until a carriage return (Enter or Return) before
it is passed to the program - this is called "cooked". This allows
certain characters to be processed (see stty(1)), such as Cntl-D,
Cntl-S, Ctrl-U Backspace); essentially rudimentary line-editing. The
terminal driver "cooks" the characters before serving them up.
The terminal can be placed into "raw" mode where the characters are
not processed by the terminal driver, but are sent straight through
(it can be set that INTR and QUIT characters are still processed).
This allows programs like emacs and vi to use the entire screen more
easily.
You can read more about this in the "Canonical mode" section of the
termios(3) manpage.
See e.g. this or this how to achieve that programmatically (did not check the code, but it should be easy to find it).
Alternatively you could use e.g. strace or ltrace to check what stty -F /dev/ttyUSB0 raw does (or read the manual page where it is described).
EDIT>
Regarding printf without a newline -- fflush(stdout); immediately after it should help (another line-buffering is taking place).
You might consider reading this and maybe this.
I want to receive an interrupt when the user presses a special keystroke like F1-12 in my program, which is written in nasm. I simply need to wait for a function keystroke at the start of my main function. I know that this is possible with the BIOS's int 16h, which returns a scancode. How can I do this under Linux?
The necessary code for this is rather complicated; I eventually figured out how to check for F1 in C with raw ioctl, read, and write. The translation to nasm should be straightforward if you're familiar with assembly and Linux syscalls.
It's not exactly what you want, in that it only checks for F1, not the rest of them. F1's sequence is 0x1b, 0x4f, 0x50. You can find other sequences with od -t x1 and pressing the key. For example, F2 is 0x1b, 0x4f, 0x51.
The basic idea is that we get the current terminal attributes, update them to be raw (cfmakeraw), and then set them back. The ioctl syscall is used for this.
On a terminal in raw mode, read() will get any character(s) the user has typed, unlike the "cooked" mode where the kernel does line-editing with backspace and control-u until the user submits the line by pressing enter or control-d (EOF).
#include <unistd.h>
#include <sys/ioctl.h>
#include <termios.h>
struct ktermios {
tcflag_t c_iflag;
tcflag_t c_oflag;
tcflag_t c_cflag;
tcflag_t c_lflag;
cc_t c_line;
cc_t c_cc[19];
};
int getch() {
unsigned char c;
read(0, &c, sizeof(c));
return c;
}
int main(int argc, char *argv[]) {
struct ktermios orig, new;
ioctl(0, TCGETS, &orig);
ioctl(0, TCGETS, &new); // or more simply new = orig;
// from cfmakeraw documentation
new.c_iflag &= ~(IGNBRK | BRKINT | PARMRK | ISTRIP | INLCR | IGNCR | ICRNL | IXON);
new.c_oflag &= ~OPOST;
new.c_lflag &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN);
new.c_cflag &= ~(CSIZE | PARENB);
new.c_cflag |= CS8;
ioctl(0, TCSETS, &new);
while (1) {
if (getch() == 0x1b && getch() == 0x4f && getch() == 0x50) {
break;
}
}
write(1, "Got F1!\n", 8);
ioctl(0, TCSETS, &orig); // restore original settings before exiting!
return 0;
}
I based this on this answer, which was very helpful.
I have been studying signals in Linux. And I've done a test program to capture SIGINT.
#include <unistd.h>
#include <signal.h>
#include <iostream>
void signal_handler(int signal_no);
int main() {
signal(SIGINT, signal_handler);
for (int i = 0; i < 10; ++i) {
std::cout << "I'm sleeping..." << std::endl;
unsigned int one_ms = 1000;
usleep(200* one_ms);
}
return 0;
}
void signal_handler(int signal_no) {
if (signal_no == SIGINT)
std::cout << "Oops, you pressed Ctrl+C!\n";
return;
}
While the output looks like this:
I'm sleeping...
I'm sleeping...
^COops, you pressed Ctrl+C!
I'm sleeping...
I'm sleeping...
^COops, you pressed Ctrl+C!
I'm sleeping...
^COops, you pressed Ctrl+C!
I'm sleeping...
^COops, you pressed Ctrl+C!
I'm sleeping...
^COops, you pressed Ctrl+C!
I'm sleeping...
I'm sleeping...
I'm sleeping...
I understand that when pressing Ctrl+C, processes in foreground process group all receives a SIGINT(if no process chooses to ignore it).
So is it that the shell(bash) AND the instance of the above program both received the signal? Where does the "^C" before each "Oops" come from?
The OS is CentOS, and the shell is bash.
It is the terminal (driver) that intercepts the ^C and translates it to a signal sent to the attached process (which is the shell) stty intr ^B would instruct the terminal driver to intercept a ^B instead. It is also the terminal driver that echoes the ^C back to the terminal.
The shell is just a process that sits at the other end of the line, and receives it's stdin from your terminal via the terminal driver (such as /dev/ttyX), and it's stdout (and stderr) are also attached to the same tty.
Note that (if echoing is enabled) the terminal sends the keystrokes to both the process (group) and back to the terminal. The stty command is just wrapper around the ioctl()s for the tty driver for the processes "controlling" tty.
UPDATE: to demonstrate that the shell is not involved, I created the following small program. It should be executed by its parent shell via exec ./a.out (it appears an interactive shell will fork a daughter shell, anyway) The program sets the key that generates the SIGINTR to ^B, switches echo off, and than waits for input from stdin.
#include <stdio.h>
#include <string.h>
#include <termios.h>
#include <unistd.h>
#include <signal.h>
#include <errno.h>
int thesignum = 0;
void handler(int signum);
void handler(int signum)
{ thesignum = signum;}
#define THE_KEY 2 /* ^B */
int main(void)
{
int rc;
struct termios mytermios;
rc = tcgetattr(0 , &mytermios);
printf("tcgetattr=%d\n", rc );
mytermios.c_cc[VINTR] = THE_KEY; /* set intr to ^B */
mytermios.c_lflag &= ~ECHO ; /* Dont echo */
rc = tcsetattr(0 , TCSANOW, &mytermios);
printf("tcsetattr(intr,%d) =%d\n", THE_KEY, rc );
printf("Setting handler()\n" );
signal(SIGINT, handler);
printf("entering pause()\n... type something followed by ^%c\n", '#'+THE_KEY );
rc = pause();
printf("Rc=%d: %d(%s), signum=%d\n", rc, errno , strerror(errno), thesignum );
// mytermios.c_cc[VINTR] = 3; /* reset intr to ^C */
mytermios.c_lflag |= ECHO ; /* Do echo */
rc = tcsetattr(0 , TCSANOW, &mytermios);
printf("tcsetattr(intr,%d) =%d\n", THE_KEY, rc );
return 0;
}
intr.sh:
#!/bin/sh
echo $$
exec ./a.out
echo I am back.
The shell echoes everything you type, so when you type ^C, that too gets echoed (and in your case intercepted by your signal handler). The command stty -echo may or may not be useful to you depending on your needs/constraints, see the man page for stty for more information.
Of course much more goes on at a lower level, anytime you communicate with a system via peripherals device drivers (such as the keyboard driver that you use to generate the ^C signal, and the terminal driver that displays everything) are involved. You can dig even deeper at the level of assembly/machine language, registers, lookup tables etc. If you want a more detailed, in-depth level of understanding the books below are a good place to start:
The Design of the Unix OS is a good reference for these sort of things. Two more classic references: Unix Programming Environment
and Advanced Programming in the UNIX Environment
Nice summary here in this SO question How does Ctrl-C terminate a child process?
"when youre run a program, for example find, the shell:
the shell fork itself
and for the child set the default signal handling
replace the child with the given command (e.g. with find)
when you press CTRL-C, parent shell handle this signal but the child will receive it - with the default action - terminate. (the child can implement signal handling too)"
I like to log a programs output 'on demand'. Eg. the output is logged to the terminal, but another process can hook on the current output at any time.
The classic way would be:
myprogram 2>&1 | tee /tmp/mylog
and on demand
tail /tmp/mylog
However, this would create a ever growing log file even if not used until the drive runs out of space. So my attempt was:
mkfifo /tmp/mylog
myprogram 2>&1 | tee /tmp/mylog
and on demand
cat /tmp/mylog
Now I can read /tmp/mylog at any time. However, any output blocks the program until the /tmp/mylog is read. I like the fifo to flush any incoming data not read back. How to do that?
Inspired by your question I've written a simple program that will let you do this:
$ myprogram 2>&1 | ftee /tmp/mylog
It behaves similarly to tee but clones the stdin to stdout and to a named pipe (a requirement for now) without blocking. This means that if you want to log this way it may happen that you're gonna lose your log data, but I guess it's acceptable in your scenario.
The trick is to block SIGPIPE signal and to ignore error on writing to a broken fifo. This sample may be optimized in various ways of course, but so far, it does the job I guess.
/* ftee - clone stdin to stdout and to a named pipe
(c) racic#stackoverflow
WTFPL Licence */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
int main(int argc, char *argv[])
{
int readfd, writefd;
struct stat status;
char *fifonam;
char buffer[BUFSIZ];
ssize_t bytes;
signal(SIGPIPE, SIG_IGN);
if(2!=argc)
{
printf("Usage:\n someprog 2>&1 | %s FIFO\n FIFO - path to a"
" named pipe, required argument\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
readfd = open(fifonam, O_RDONLY | O_NONBLOCK);
if(-1==readfd)
{
perror("ftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status))
{
perror("ftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode))
{
printf("ftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK);
if(-1==writefd)
{
perror("ftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd);
while(1)
{
bytes = read(STDIN_FILENO, buffer, sizeof(buffer));
if (bytes < 0 && errno == EINTR)
continue;
if (bytes <= 0)
break;
bytes = write(STDOUT_FILENO, buffer, bytes);
if(-1==bytes)
perror("ftee: writing to stdout");
bytes = write(writefd, buffer, bytes);
if(-1==bytes);//Ignoring the errors
}
close(writefd);
return(0);
}
You can compile it with this standard command:
$ gcc ftee.c -o ftee
You can quickly verify it by running e.g.:
$ ping www.google.com | ftee /tmp/mylog
$ cat /tmp/mylog
Also note - this is no multiplexer. You can only have one process doing $ cat /tmp/mylog at a time.
This is a (very) old thread, but I've run into a similar problem of late. In fact, what I needed is a cloning of stdin to stdout with a copy to a pipe that is non blocking. the proposed ftee in the first answer really helped there, but was (for my use case) too volatile. Meaning I lost data I could have processed if I had gotten to it in time.
The scenario I was faced with is that I have a process (some_process) that aggregates some data and writes its results every three seconds to stdout. The (simplified) setup looked like this (in the real setup I am using a named pipe):
some_process | ftee >(onlineAnalysis.pl > results) | gzip > raw_data.gz
Now, raw_data.gz has to be compressed and has to be complete. ftee does this job very well. But the pipe I am using in the middle was too slow to grab the data flushed out - but it was fast enough to process everything if it could get to it, which was tested with a normal tee. However, a normal tee blocks if anything happens to the unnamed pipe, and as I want to be able to hook in on demand, tee is not an option. Back to the topic: It got better when I put a buffer in between, resulting in:
some_process | ftee >(mbuffer -m 32M| onlineAnalysis.pl > results) | gzip > raw_data.gz
But that was still losing data I could have processed. So I went ahead and extended the ftee proposed before to a buffered version (bftee). It still has all the same properties, but uses an (inefficient ?) internal buffer in case a write fails. It still loses data if the buffer runs full, but it works beautifully for my case. As always there is a lot of room for improvement, but as I copied the code off of here I'd like to share it back to people that might have a use for it.
/* bftee - clone stdin to stdout and to a buffered, non-blocking pipe
(c) racic#stackoverflow
(c) fabraxias#stackoverflow
WTFPL Licence */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
// the number of sBuffers that are being held at a maximum
#define BUFFER_SIZE 4096
#define BLOCK_SIZE 2048
typedef struct {
char data[BLOCK_SIZE];
int bytes;
} sBuffer;
typedef struct {
sBuffer *data; //array of buffers
int bufferSize; // number of buffer in data
int start; // index of the current start buffer
int end; // index of the current end buffer
int active; // number of active buffer (currently in use)
int maxUse; // maximum number of buffers ever used
int drops; // number of discarded buffer due to overflow
int sWrites; // number of buffer written to stdout
int pWrites; // number of buffers written to pipe
} sQueue;
void InitQueue(sQueue*, int); // initialized the Queue
void PushToQueue(sQueue*, sBuffer*, int); // pushes a buffer into Queue at the end
sBuffer *RetrieveFromQueue(sQueue*); // returns the first entry of the buffer and removes it or NULL is buffer is empty
sBuffer *PeakAtQueue(sQueue*); // returns the first entry of the buffer but does not remove it. Returns NULL on an empty buffer
void ShrinkInQueue(sQueue *queue, int); // shrinks the first entry of the buffer by n-bytes. Buffer is removed if it is empty
void DelFromQueue(sQueue *queue); // removes the first entry of the queue
static void sigUSR1(int); // signal handled for SUGUSR1 - used for stats output to stderr
static void sigINT(int); // signla handler for SIGKILL/SIGTERM - allows for a graceful stop ?
sQueue queue; // Buffer storing the overflow
volatile int quit; // for quiting the main loop
int main(int argc, char *argv[])
{
int readfd, writefd;
struct stat status;
char *fifonam;
sBuffer buffer;
ssize_t bytes;
int bufferSize = BUFFER_SIZE;
signal(SIGPIPE, SIG_IGN);
signal(SIGUSR1, sigUSR1);
signal(SIGTERM, sigINT);
signal(SIGINT, sigINT);
/** Handle commandline args and open the pipe for non blocking writing **/
if(argc < 2 || argc > 3)
{
printf("Usage:\n someprog 2>&1 | %s FIFO [BufferSize]\n"
"FIFO - path to a named pipe, required argument\n"
"BufferSize - temporary Internal buffer size in case write to FIFO fails\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
if (argc == 3) {
bufferSize = atoi(argv[2]);
if (bufferSize == 0) bufferSize = BUFFER_SIZE;
}
readfd = open(fifonam, O_RDONLY | O_NONBLOCK);
if(-1==readfd)
{
perror("bftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status))
{
perror("bftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode))
{
printf("bftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK);
if(-1==writefd)
{
perror("bftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd);
InitQueue(&queue, bufferSize);
quit = 0;
while(!quit)
{
// read from STDIN
bytes = read(STDIN_FILENO, buffer.data, sizeof(buffer.data));
// if read failed due to interrupt, then retry, otherwise STDIN has closed and we should stop reading
if (bytes < 0 && errno == EINTR) continue;
if (bytes <= 0) break;
// save the number if read bytes in the current buffer to be processed
buffer.bytes = bytes;
// this is a blocking write. As long as buffer is smaller than 4096 Bytes, the write is atomic to a pipe in Linux
// thus, this cannot be interrupted. however, to be save this should handle the error cases of partial or interrupted write none the less.
bytes = write(STDOUT_FILENO, buffer.data, buffer.bytes);
queue.sWrites++;
if(-1==bytes) {
perror("ftee: writing to stdout");
break;
}
sBuffer *tmpBuffer = NULL;
// if the queue is empty (tmpBuffer gets set to NULL) the this does nothing - otherwise it tries to write
// the buffered data to the pipe. This continues until the Buffer is empty or the write fails.
// NOTE: bytes cannot be -1 (that would have failed just before) when the loop is entered.
while ((bytes != -1) && (tmpBuffer = PeakAtQueue(&queue)) != NULL) {
// write the oldest buffer to the pipe
bytes = write(writefd, tmpBuffer->data, tmpBuffer->bytes);
// the written bytes are equal to the buffer size, the write is successful - remove the buffer and continue
if (bytes == tmpBuffer->bytes) {
DelFromQueue(&queue);
queue.pWrites++;
} else if (bytes > 0) {
// on a positive bytes value there was a partial write. we shrink the current buffer
// and handle this as a write failure
ShrinkInQueue(&queue, bytes);
bytes = -1;
}
}
// There are several cases here:
// 1.) The Queue is empty -> bytes is still set from the write to STDOUT. in this case, we try to write the read data directly to the pipe
// 2.) The Queue was not empty but is now -> bytes is set from the last write (which was successful) and is bigger 0. also try to write the data
// 3.) The Queue was not empty and still is not -> there was a write error before (even partial), and bytes is -1. Thus this line is skipped.
if (bytes != -1) bytes = write(writefd, buffer.data, buffer.bytes);
// again, there are several cases what can happen here
// 1.) the write before was successful -> in this case bytes is equal to buffer.bytes and nothing happens
// 2.) the write just before is partial or failed all together - bytes is either -1 or smaller than buffer.bytes -> add the remaining data to the queue
// 3.) the write before did not happen as the buffer flush already had an error. In this case bytes is -1 -> add the remaining data to the queue
if (bytes != buffer.bytes)
PushToQueue(&queue, &buffer, bytes);
else
queue.pWrites++;
}
// once we are done with STDIN, try to flush the buffer to the named pipe
if (queue.active > 0) {
//set output buffer to block - here we wait until we can write everything to the named pipe
// --> this does not seem to work - just in case there is a busy loop that waits for buffer flush aswell.
int saved_flags = fcntl(writefd, F_GETFL);
int new_flags = saved_flags & ~O_NONBLOCK;
int res = fcntl(writefd, F_SETFL, new_flags);
sBuffer *tmpBuffer = NULL;
//TODO: this does not handle partial writes yet
while ((tmpBuffer = PeakAtQueue(&queue)) != NULL) {
int bytes = write(writefd, tmpBuffer->data, tmpBuffer->bytes);
if (bytes != -1) DelFromQueue(&queue);
}
}
close(writefd);
}
/** init a given Queue **/
void InitQueue (sQueue *queue, int bufferSize) {
queue->data = calloc(bufferSize, sizeof(sBuffer));
queue->bufferSize = bufferSize;
queue->start = 0;
queue->end = 0;
queue->active = 0;
queue->maxUse = 0;
queue->drops = 0;
queue->sWrites = 0;
queue->pWrites = 0;
}
/** push a buffer into the Queue**/
void PushToQueue(sQueue *queue, sBuffer *p, int offset)
{
if (offset < 0) offset = 0; // offset cannot be smaller than 0 - if that is the case, we were given an error code. Set it to 0 instead
if (offset == p->bytes) return; // in this case there are 0 bytes to add to the queue. Nothing to write
// this should never happen - offset cannot be bigger than the buffer itself. Panic action
if (offset > p->bytes) {perror("got more bytes to buffer than we read\n"); exit(EXIT_FAILURE);}
// debug output on a partial write. TODO: remove this line
// if (offset > 0 ) fprintf(stderr, "partial write to buffer\n");
// copy the data from the buffer into the queue and remember its size
memcpy(queue->data[queue->end].data, p->data + offset , p->bytes-offset);
queue->data[queue->end].bytes = p->bytes - offset;
// move the buffer forward
queue->end = (queue->end + 1) % queue->bufferSize;
// there is still space in the buffer
if (queue->active < queue->bufferSize)
{
queue->active++;
if (queue->active > queue->maxUse) queue->maxUse = queue->active;
} else {
// Overwriting the oldest. Move start to next-oldest
queue->start = (queue->start + 1) % queue->bufferSize;
queue->drops++;
}
}
/** return the oldest entry in the Queue and remove it or return NULL in case the Queue is empty **/
sBuffer *RetrieveFromQueue(sQueue *queue)
{
if (!queue->active) { return NULL; }
queue->start = (queue->start + 1) % queue->bufferSize;
queue->active--;
return &(queue->data[queue->start]);
}
/** return the oldest entry in the Queue or NULL if the Queue is empty. Does not remove the entry **/
sBuffer *PeakAtQueue(sQueue *queue)
{
if (!queue->active) { return NULL; }
return &(queue->data[queue->start]);
}
/*** Shrinks the oldest entry i the Queue by bytes. Removes the entry if buffer of the oldest entry runs empty*/
void ShrinkInQueue(sQueue *queue, int bytes) {
// cannot remove negative amount of bytes - this is an error case. Ignore it
if (bytes <= 0) return;
// remove the entry if the offset is equal to the buffer size
if (queue->data[queue->start].bytes == bytes) {
DelFromQueue(queue);
return;
};
// this is a partial delete
if (queue->data[queue->start].bytes > bytes) {
//shift the memory by the offset
memmove(queue->data[queue->start].data, queue->data[queue->start].data + bytes, queue->data[queue->start].bytes - bytes);
queue->data[queue->start].bytes = queue->data[queue->start].bytes - bytes;
return;
}
// panic is the are to remove more than we have the buffer
if (queue->data[queue->start].bytes < bytes) {
perror("we wrote more than we had - this should never happen\n");
exit(EXIT_FAILURE);
return;
}
}
/** delete the oldest entry from the queue. Do nothing if the Queue is empty **/
void DelFromQueue(sQueue *queue)
{
if (queue->active > 0) {
queue->start = (queue->start + 1) % queue->bufferSize;
queue->active--;
}
}
/** Stats output on SIGUSR1 **/
static void sigUSR1(int signo) {
fprintf(stderr, "Buffer use: %i (%i/%i), STDOUT: %i PIPE: %i:%i\n", queue.active, queue.maxUse, queue.bufferSize, queue.sWrites, queue.pWrites, queue.drops);
}
/** handle signal for terminating **/
static void sigINT(int signo) {
quit++;
if (quit > 1) exit(EXIT_FAILURE);
}
This version takes one more (optional) argument which specifies the number of the blocks that are to buffered for the pipe. My sample call now looks like this:
some_process | bftee >(onlineAnalysis.pl > results) 16384 | gzip > raw_data.gz
resulting in 16384 blocks to be buffered before discards happen. this uses about 32 Mbyte more memory, but... who cares ?
Of course, in the real environment I am using a named pipe so that I can attach and detach as needed. There is looks like this:
mkfifo named_pipe
some_process | bftee named_pipe 16384 | gzip > raw_data.gz &
cat named_pipe | onlineAnalysis.pl > results
Also, the process reacts on signals as follows:
SIGUSR1 -> print counters to STDERR
SIGTERM, SIGINT -> first exits the main loop and flushed the buffer to the pipe, the second terminated the program immediatly.
Maybe this helps someone in the future...
Enjoy
However, this would create a ever growing log file even if not used until the drive runs out of space.
Why not periodically rotate the logs? There's even a program to do it for you logrotate.
There's also a system for generating log messages and doing different things with them according to type. It's called syslog.
You could even combine the two. Have your program generate syslog messages, configure syslog to place them in a file and use logrotate to ensure they don't fill the disk.
If it turned out that you were writing for a small embedded system and the program's output is heavy there are a variety of techniques you might consider.
Remote syslog: send the syslog messages to a syslog server on the network.
Use the severity levels availble in syslog to do different things with the messages. E.g. discard "INFO" but log and forward "ERR" or greater. E.g. to console
Use a signal handler in your program to reread configuration on HUP and vary log generation "on demand" this way.
Have your program listen on a unix socket and write messages down it when open. You could even implement and interactive console into your program this way.
Using a configuration file, provide granular control of logging output.
It seems like bash <> redirection operator (3.6.10 Opening File Descriptors for Reading and WritingSee) makes writing to file/fifo opened with it non-blocking.
This should work:
$ mkfifo /tmp/mylog
$ exec 4<>/tmp/mylog
$ myprogram 2>&1 | tee >&4
$ cat /tmp/mylog # on demend
Solution given by gniourf_gniourf on #bash IRC channel.
BusyBox often used on embedded devices can create a ram buffered log by
syslogd -C
which can be filled by
logger
and read by
logread
Works quite well, but only provides one global log.
The logging could be directed to a UDP socket. Since UDP is connection-less, it won't block the sending program. Of course logs will be lost if the receiver or network can't keep up.
myprogram 2>&1 | socat - udp-datagram:localhost:3333
Then when you want to observe the logging:
socat udp-recv:3333 -
There are some other cool benefits like being able to attach multiple listeners at the same time or broadcast to multiple devices.
If you can install screen on the embedded device then you can run 'myprogram' in it and detach it, and reattach it anytime you want to see the log. Something like:
$ screen -t sometitle myprogram
Hit Ctrl+A, then d to detach it.
Whenever you want to see the output, reattach it:
$ screen -DR sometitle
Hit Ctrl-A, then d to detach it again.
This way you won't have to worry about the program output using disk space at all.
The problem with the given fifo approach is that the whole thing will hang when the pipe buffer is getting filled up and no reading process is taking place.
For the fifo approach to work I think you would have to implement a named pipe client-server model similar to the one mentioned in BASH: Best architecture for reading from two input streams (see slightly modified code below, sample code 2).
For a workaround you could also use a while ... read construct instead of teeing stdout to a named pipe by implementing a counting mechanism inside the while ... read loop that will overwrite the log file periodically by a specified number of lines. This would prevent an ever growing log file (sample code 1).
# sample code 1
# terminal window 1
rm -f /tmp/mylog
touch /tmp/mylog
while sleep 2; do date '+%Y-%m-%d_%H.%M.%S'; done 2>&1 | while IFS="" read -r line; do
lno=$((lno+1))
#echo $lno
array[${lno}]="${line}"
if [[ $lno -eq 10 ]]; then
lno=$((lno+1))
array[${lno}]="-------------"
printf '%s\n' "${array[#]}" > /tmp/mylog
unset lno array
fi
printf '%s\n' "${line}"
done
# terminal window 2
tail -f /tmp/mylog
#------------------------
# sample code 2
# code taken from:
# https://stackoverflow.com/questions/6702474/bash-best-architecture-for-reading-from-two-input-streams
# terminal window 1
# server
(
rm -f /tmp/to /tmp/from
mkfifo /tmp/to /tmp/from
while true; do
while IFS="" read -r -d $'\n' line; do
printf '%s\n' "${line}"
done </tmp/to >/tmp/from &
bgpid=$!
exec 3>/tmp/to
exec 4</tmp/from
trap "kill -TERM $bgpid; exit" 0 1 2 3 13 15
wait "$bgpid"
echo "restarting..."
done
) &
serverpid=$!
#kill -TERM $serverpid
# client
(
exec 3>/tmp/to;
exec 4</tmp/from;
while IFS="" read -r -d $'\n' <&4 line; do
if [[ "${line:0:1}" == $'\177' ]]; then
printf 'line from stdin: %s\n' "${line:1}" > /dev/null
else
printf 'line from fifo: %s\n' "$line" > /dev/null
fi
done &
trap "kill -TERM $"'!; exit' 1 2 3 13 15
while IFS="" read -r -d $'\n' line; do
# can we make it atomic?
# sleep 0.5
# dd if=/tmp/to iflag=nonblock of=/dev/null # flush fifo
printf '\177%s\n' "${line}"
done >&3
) &
# kill -TERM $!
# terminal window 2
# tests
echo hello > /tmp/to
yes 1 | nl > /tmp/to
yes 1 | nl | tee /tmp/to
while sleep 2; do date '+%Y-%m-%d_%H.%M.%S'; done 2>&1 | tee -a /tmp/to
# terminal window 3
cat /tmp/to | head -n 10
If your process writes to any log file and then wipes the file and starts again every now and again, so it doesn't get too big, or uses logrotate.
tail --follow=name --retry my.log
Is all you need. You will get as much scroll-back as your terminal.
Nothing non standard is needed. I've not tried it with small log files but all our logs rotate like this and I have never noticed loosing lines.
To follow in Fabraxias foot steps I'm going to share my small modification of racic's code. In one of my use cases I needed to suppress the writes to STDOUT, so I've added another parameter: swallow_stdout. If that is not 0, then output to STDOUT will be turned off.
Since I'm no C coder I've added comments while reading the code, maybe they are useful for others.
/* ftee - clone stdin to stdout and to a named pipe
(c) racic#stackoverflow
WTFPL Licence */
// gcc /tmp/ftee.c -o /usr/local/bin/ftee
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
int main(int argc, char *argv[])
{
int readfd, writefd; // read & write file descriptors
struct stat status; // read file descriptor status
char *fifonam; // name of the pipe
int swallow_stdout; // 0 = write to STDOUT
char buffer[BUFSIZ]; // read/write buffer
ssize_t bytes; // bytes read/written
signal(SIGPIPE, SIG_IGN);
if(3!=argc)
{
printf("Usage:\n someprog 2>&1 | %s [FIFO] [swallow_stdout] \n"
"FIFO - path to a named pipe (created beforehand with mkfifo), required argument\n"
"swallow_stdout - 0 = output to PIPE and STDOUT, 1 = output to PIPE only, required argument\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
swallow_stdout = atoi(argv[2]);
readfd = open(fifonam, O_RDONLY | O_NONBLOCK); // open read file descriptor in non-blocking mode
if(-1==readfd) // read descriptor error!
{
perror("ftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status)) // read descriptor status error! (?)
{
perror("ftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode)) // read descriptor is not a FIFO error!
{
printf("ftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK); // open write file descriptor non-blocking
if(-1==writefd) // write file descriptor error!
{
perror("ftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd); // reading complete, close read file descriptor
while(1) // infinite loop
{
bytes = read(STDIN_FILENO, buffer, sizeof(buffer)); // read STDIN into buffer
if (bytes < 0 && errno == EINTR)
continue; // skip over errors
if (bytes <= 0)
break; // no more data coming in or uncaught error, let's quit since we can't write anything
if (swallow_stdout == 0)
bytes = write(STDOUT_FILENO, buffer, bytes); // write buffer to STDOUT
if(-1==bytes) // write error!
perror("ftee: writing to stdout");
bytes = write(writefd, buffer, bytes); // write a copy of the buffer to the write file descriptor
if(-1==bytes);// ignore errors
}
close(writefd); // close write file descriptor
return(0); // return exit code 0
}