How do I get an equivalent of /dev/one in Linux - linux

You can use
dd if=/dev/zero of=file count=1024 bs=1024
to zero fill a file.
Instead of that I want to one fill a file. How do I do that?
There is no /dev/one file, so how can I simulate that effect via on bash shell?

tr '\0' '\377' < /dev/zero | dd bs=64K of=/dev/sdx
This should be much faster. Choose your blocksizes (or add counts) like you need at. Writing ones to a SSD-Disk till full with a blocksize of 99M gave me 350M/s write performance.

Try this:
dd if=<(yes $'\01' | tr -d "\n") of=file count=1024 bs=1024
Substitute $'\377' or $'\xFF' if you want all the bits to be ones.
MacOS tr may complain about "Illegal byte sequence". Setting LC_CTYPE=C will prevent that. This version can also be used in Linux:
dd if=<(yes $'\01' | LC_CTYPE=C tr -d "\n") of=file count=1024 bs=1024

Well, you could do this:
dd if=/dev/zero count=1024 bs=1024 |
tr '\000' '\001' > file

pv /dev/zero |tr \\000 \\377 >targetfile
...where \377 is the octal representation of 255 (a byte with all bits set to one). Why tr only works with octal numbers, I don't know -- but be careful not to subconsciously translate this to 3FF.
The syntax for using tr is error prone. I recommend verifying that it is making the desired translation...
cat /dev/zero |tr \\000 \\377 |hexdump -C
Note: pv is a nice utility that replaces cat and adds a progress/rate display.

I created a device driver in my github. Installing it creates a file /dev/one that is writing only bits set to 1.
The c file called one.c (the only interesting part is in device_file_read):
// File Driver to create a devince /dev/one like the /dev/zero
#include <linux/init.h>
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
MODULE_LICENSE("GPL");
static int device_file_major_number = 0;
static const char device_name[] = "one";
static ssize_t device_file_read(
struct file *file_ptr,
char __user *user_buffer,
size_t count,
loff_t *position) {
printk( KERN_NOTICE "One: Device file is read at offset = %i, read bytes count = %u\n" , (int)*position , (unsigned int)count );
// Allocate Kernel buffer
char* ptr = (char*) vmalloc(count);
// Fill it with one, byte per byte
// -- Note that byte is the smallest accesible data unit
memset(ptr, 0xFF, count);
char res = copy_to_user(user_buffer, ptr, count);
if (res != 0){ return -EFAULT; }
// Return number of byte read
return count;
}
static struct file_operations simple_driver_fops = {
.owner = THIS_MODULE,
.read = device_file_read,
};
int register_device(void) {
int res = 0;
printk( KERN_NOTICE "One: register_device() is called.\n" );
res = register_chrdev( 0, device_name, &simple_driver_fops );
if( res < 0 ) {
printk( KERN_WARNING "One: can\'t register character device with error code = %i\n", res );
return res;
}
device_file_major_number = res;
printk( KERN_NOTICE "One: registered character device with major number = %i and minor numbers 0...255\n", device_file_major_number );
return 0;
}
void unregister_device(void) {
printk( KERN_NOTICE "One: unregister_device() is called\n" );
if(device_file_major_number != 0) {
unregister_chrdev(device_file_major_number, device_name);
}
}
static int my_init(void) {
register_device();
return 0;
}
static void my_exit(void) {
unregister_device();
return;
}
// Declare register and unregister command
module_init(my_init);
module_exit(my_exit);
The Makefile
TARGET_MODULE:=one
BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build
PWD:=$(shell pwd)
obj-m := $(TARGET_MODULE).o
# See: https://stackoverflow.com/questions/15910064/how-to-compile-a-linux-kernel-module-using-std-gnu99
ccflags-y := -std=gnu99 -Wno-declaration-after-statement
build:
# run kernel build system to make module
$(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules
clean:
# run kernel build system to cleanup in current directory
$(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean
rm -f MOK.priv MOK*.der
key:
echo "Creating key"
openssl req -new -x509 -newkey rsa:2048 -days 36500 -keyout MOK.priv -outform DER -out MOK.der -nodes -subj "/CN=TinmarinoUnsafe/"
#
echo "\e[31;1mPlease enter a password you will be asked for on reboot:\e[0m"
mokutil --import MOK.der
echo "\e[31;1mNow you must: 1/ reboot, 2/ Select Unroll MOK, 3/ Enter password you previously gave\e[0m"
sign:
cp one.ko one.ko.bck
/usr/src/linux-headers-$(shell uname -r)/scripts/sign-file sha256 MOK.priv MOK.der one.ko
load:
insmod ./$(TARGET_MODULE).ko
unload:
rmmod ./$(TARGET_MODULE).ko
create:
mknod /dev/one c $(shell cat /proc/devices | grep one$ | cut -d ' ' -f1) 0
delete:
rm /dev/one
test:
[ "$(shell xxd -p -l 10 /dev/one)" = "ffffffffffffffffffff" ] \
&& echo "\e[32mSUCCESS\e[0m" \
|| echo "\e[31mFAILED\e[0m"
The instalation is long (3min) due to the driver signature enforcement. Froget this part if you disabled it in your UEFI.
git clone https://github.com/tinmarino/dev_one.git DevOne && cd DevOne # Download
make build # Compile
make key # Generate key for signing
sudo make sign # Sign driver module to permit MOK enforcement (security)
sudo reboot now # Reboot and enable Mok
A blue screen (MOK manager) will appear
Choose "Enroll MOK"
Choose "Continue"
Choose "Yes" (when asked "Enroll the key")
Enter the password you gave at make sign
Choose "Reboot" (again)
sudo make load # Load
sudo make device # Create /dev/one
make test # Test if all is ok

You can simulate a /dev/one without a special device, with a FIFO + yes:
mkfifo ddfifo
dd if=ddfifo of=<file> iflag=fullblock count=1024 bs=1024 status=progress & yes "" | tr '\n' '\1' > ddfifo
tee may be used to double the throughput:
mkfifo ddfifo
dd if=ddfifo of=<file> iflag=fullblock count=1024 bs=1024 status=progress & yes "" | tr '\n' '\1' | tee ddfifo > ddfifo
If you'd like bytes with all bits set to one, swap '\1' for '\377'.

Related

Why does my RIP value change after overwriting via an overflow?

I've been working on a buffer overflow on a 64 bit Linux machine for the past few days. The code I'm attacking takes in a file. This original homework ran on a 32-bit system, so a lot is differing. I thought I'd run with it and try to learn something new along the way. I set sudo sysctl -w kernel.randomize_va_space=0 and compiled the program below gcc -o stack -z execstack -fno-stack-protector stack.c && chmod 4755 stack
/* This program has a buffer overflow vulnerability. */
/* Our task is to exploit this vulnerability */
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
int bof(char *str)
{
char buffer[12];
/* The following statement has a buffer overflow problem */
strcpy(buffer, str);
return 1;
}
int main(int argc, char **argv)
{
char str[517];
FILE *badfile;
badfile = fopen("badfile", "r");
fread(str, sizeof(char), 517, badfile);
bof(str);
printf("Returned Properly\n");
return 1;
}
I could get it to crash by making a file with 20 "A"s. I made a small script to help.
#!/usr/bin/bash
python -c 'print("A"*20)' | tr -d "\n" > badfile
So now, if I add 6 "B"s to the mix after hitting the SIGSEGV in gbd I get.
RIP: 0x424242424242 ('BBBBBB')
0x0000424242424242 in ?? ()
Perfect! Now we can play with the RIP and put our address in for it to jump to! This is where I'm getting a little confused.
I added some C's to the script to try to find a good address to jump to
python -c 'print("A"*20 + "B"*6 + C*32)' | tr -d "\n" > badfile
after creating the file and getting the SEGFAULT my RIP address is no longer our B's
RIP: 0x55555555518a (<bof+37>: ret)
Stopped reason: SIGSEGV
0x000055555555518a in bof (
str=0x7fffffffe900 'C' <repeats 14 times>)
at stack.c:16
I thought this might be due to using B's, so I went ahead and tried to find an address. I then ran x/100 $rsp in gbd, but it looks completely different than before without the Cs
# Before Cs
0x7fffffffe8f0: 0x00007fffffffec08 0x0000000100000000
0x7fffffffe900: 0x4141414141414141 0x4141414141414141
0x7fffffffe910: 0x4242424241414141 0x0000000000004242
# After Cs
0x7fffffffe8e8: "BBBBBB", 'C' <repeats 32 times>
0x7fffffffe90f: "AAAAABBBBBB", 'C' <repeats 32 times>
0x7fffffffe93b: ""
I've been trying to understand why this is happening. I know after this I can supply an address plus code to get a shell like so
python -c 'print("noOPs"*20 + "address" + "shellcode")' | tr -d "\n" > badfile
The only thing that has come to mind is the buffer size? I'm not too sure, though. Any help would be great. Doing this alone without help has made me learn a lot. I'm just dying to create a working exploit!

iotop does not show any disk read statistics

I am running a test to check disk read statistics. Here is the code for the same:
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
int main(int argc, char* argv)
{
int count=1000, size;
char block[4096]="0";
int fd = open("file1.txt",O_RDONLY | O_SYNC);
//int pid = getpid();
system("pid=$(ps -a | grep 'a.out' | awk '{print $1}'); iotop -bokp $pid > test1c.out &");
system("echo 'Starts reading in 10'");
srand(time(NULL));
system("sleep 1");
while(count--){
int random = (rand()%16)*666;
printf("%d;",random);
lseek(fd, random, SEEK_SET);
size = read(fd,block,4096);
printf("Number of bytes read: %d\n", size);
fsync(fd);
//printf("Read 4kb from the file.\n");
}
system("sleep 1");
system("killall iotop");
}
As you can see, I am opening a large file, getting the PID of my a.out file, and passing it to iotop. After that I am randomly seeking to a 4kb block in the file and reading data.
If you run this code on your system, you'll realize that iotop output shows 0kb reads throughout, which makes no sense. Am I doing something wrong?
Clearing the caches solved the issue. I found the script for clearing caches on this page:
https://www.tecmint.com/clear-ram-memory-cache-buffer-and-swap-space-on-linux/
sync; echo 1 > /proc/sys/vm/drop_caches
sync; echo 2 > /proc/sys/vm/drop_caches
sync; echo 3 > /proc/sys/vm/drop_caches
Does the trick!

opencv videocapture can't open MJPEG stream

the Opencv 2.4.9: VideoCapture cann't open the MJPG-streamer:
VideoCapture cap;
cap.open("http://127.0.0.1:8080/?action=stream&type=.mjpg");
if (!cap.isOpened()) // if not success, exit program
{
cout << "Cannot open the video cam" << endl;
return -1;
}
I can see the video use gst-launch. and I have searched a lot,like this and tried the fifo like this,but still cann't open it.
then I want to debug into opencv,compile opencv with CMAKE_BUILD_TYPE=DEBUG, but my GDB just cann't step into the open function.any idea?
my makefile:
OpencvDebugLibDir=/home/ry/lib
CFLAGS = -g -I$(OpencvDebugLibDir)/include/opencv -I$(OpencvDebugLibDir)
LIBS = $(OpencvDebugLibDir)/lib/*.so
target : main.cpp
g++ $(CFLAGS) $(LIBS) -o $# $<
by the way, I am in opensue13.1, with the same code,I can open the video in win 7.
thank you.
Update
now I can step into some function like:imshow,waitKey, but I can not step into others like:imread,namedWindow,it shows:
29 image = cv::imread(name);
(gdb) s
std::allocator<char>::allocator (this=0x7fffffffdc7f)
at /usr/src/debug/gcc-4.8.1-20130909/obj-x86_64-suse-linux/x86_64-suse-linux/libstdc++-v3/include/bits/allocator.h:113
113 allocator() throw() { }
test4.cpp:
#include <stdio.h>
#include <opencv2/opencv.hpp>
using namespace cv;
int main(int argc, char** argv )
{
Mat image;
image = imread( "LinuxLogo.jpg", 1 );
if ( !image.data )
{
printf("No image data \n");
return -1;
}
namedWindow("Display Image", CV_WINDOW_AUTOSIZE );
imshow("Display Image", image);
waitKey(0);
return 0;
}
my makefile:
OpencvDebugLibDir=/home/ry/lib
CFLAGS=-g -I$(OpencvDebugLibDir)/include/opencv -I$(OpencvDebugLibDir)
LIBS=$(OpencvDebugLibDir)/lib
test4:test4.cpp
g++ $(CFLAGS) -o $# $< -L$(LIBS) -lopencv_highgui -lopencv_core -Wl,-rpath=/home/ry/lib/lib
run gdb:
gdb test4 -d /home/ry/learn/opencv/install/OpenCV/opencv-2.4.9/modules/core/src -d /home/ry/learn/opencv/install/OpenCV/opencv-2.4.9/modules/highgui/src
opencv videocapture can't open MJPEG stream,because I don't compile opencv with FFMPEG support.
detail:
when cmake the opencv:
cmake -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local -D BUILD_SHARED_LIBS=ON -D WITH_TBB=ON -D WITH_OPENMP=ON -D WITH_OPENCL=ON -D WITH_CUDA=ON -D WITH_GTK=ON -D BUILD_NEW_PYTHON_SUPPORT=ON -D WITH_V4L=ON -D INSTALL_C_EXAMPLES=OFF -D INSTALL_PYTHON_EXAMPLES=OFF -D BUILD_EXAMPLES=OFF -D WITH_QT=ON -D WITH_OPENGL=ON -D BUILD_JPEG=ON -D BUILD_PNG=ON -D BUILD_JASPER=ON -D BUILD_ZLIB=ON -D WITH_JPEG=ON -D WITH_PNG=ON -D WITH_JASPER=ON -D WITH_ZLIB=ON -D WITH_OPENEXR=OFF ..
you get something like:
FFMPEG: NO
-- codec: NO
-- format: NO
-- util: NO
-- swscale: NO
-- gentoo-style: NO
so the cmake don't find the FFMPEG. I should install libffmpeg-devel in my machine(Opensuse 13.1),then the pkg-config can find FFMPEG,you can check with this:
pkg-config --list-all | grep libavcodec
then run the above cmake command again, I get:
FFMPEG: YES
-- codec: YES (ver 55.69.100)
-- format: YES (ver 55.48.100)
-- util: YES (ver 52.92.100)
-- swscale: YES (ver 2.6.100)
-- gentoo-style: YES
make,and I get the opencv videocapture able to open MJPG_Streamer.
PS:to find the reason,I compile a debug version opencv,and step into the VideoCapture's open function,and in the construction function of icvInitFFMPEG() in " opencv-2.4.9/modules/highgui/src/cap.cpp ":
#elif defined HAVE_FFMPEG
icvCreateFileCapture_FFMPEG_p = (CvCreateFileCapture_Plugin)cvCreateFileCapture_FFMPEG;
icvReleaseCapture_FFMPEG_p = (CvReleaseCapture_Plugin)cvReleaseCapture_FFMPEG;
icvGrabFrame_FFMPEG_p = (CvGrabFrame_Plugin)cvGrabFrame_FFMPEG;
icvRetrieveFrame_FFMPEG_p = (CvRetrieveFrame_Plugin)cvRetrieveFrame_FFMPEG;
icvSetCaptureProperty_FFMPEG_p = (CvSetCaptureProperty_Plugin)cvSetCaptureProperty_FFMPEG;
icvGetCaptureProperty_FFMPEG_p = (CvGetCaptureProperty_Plugin)cvGetCaptureProperty_FFMPEG;
icvCreateVideoWriter_FFMPEG_p = (CvCreateVideoWriter_Plugin)cvCreateVideoWriter_FFMPEG;
icvReleaseVideoWriter_FFMPEG_p = (CvReleaseVideoWriter_Plugin)cvReleaseVideoWriter_FFMPEG;
icvWriteFrame_FFMPEG_p = (CvWriteFrame_Plugin)cvWriteFrame_FFMPEG;
#endif
it just step over these code,so I know because I don't have the HAVE_FFMPEG defined in compiling process.

How to load LUKS passphrase from USB, falling back to keyboard?

I want to set up a headless Linux (Debian Wheezy) PC with whole disk encryption, with the ability to unlock the disk either with a USB drive, or by entering a passphrase by keyboard. My starting point is a fresh install using the basic whole disk encryption option in the Debian Installer, which manages everything besides /boot as a LUKS-encrypted logical volume group and gives me the keyboard option. I will describe my current solution in an answer, in hopes that it will be useful and that others can improve on it.
Here are some of the issues I had:
Setting up a passphrase and putting it on the USB drive.
Loading the USB modules in time.
Waiting for the USB drive to recognized by Linux before trying to read from it.
Identifying the correct USB drive (not some other drive that happens to be inserted).
Writing a "keyscript" to pull a passphrase off the USB drive.
Ensuring that the fall-back to keyboard kicks in in all USB failure cases.
I will accept an answer with significant improvements and upvote answers that offer contributions.
A lot of my solution is derived from the post, Using A USB Key For The LUKS Passphrase.
Create a random passphrase:
dd if=/dev/urandom bs=1 count=256 > passphrase
Insert a USB drive. dmesg output will show the device name; assume /dev/sdd. Figure out its size:
blockdev --getsize64 /dev/sdd
I decided to install the passphrase at the end of the raw device, figuring it might survive any accidental use of the USB drive.
dd if=passphrase of=/dev/sdd bs=1 seek=<size-256>
Add the passphrase to the LUKS volume:
cryptsetup luksAddKey /dev/sda5 passphrase
This does not affect the existing hand-entered passphrase from the installer. The passphrase file can be deleted:
rm passphrase
Find a unique name for the USB stick, so we can identify it when present:
ls -l /dev/disk/by-id | grep -w sdd
You should see one symlink. I will call it /dev/disk/by-id/<ID>.
Edit /etc/crypttab. You should see a line like:
sdc5_crypt UUID=b9570e0f-3bd3-40b0-801f-ee20ac460207 none luks
Modify it to:
sdc5_crypt UUID=b9570e0f-3bd3-40b0-801f-ee20ac460207 /dev/disk/by-id/<ID> luks,keyscript=/bin/passphrase-from-usb
The keyscript referred to above will need to read the passphrase from the USB device. However, it needs to do more than that. To understand how it is used, check /usr/share/initramfs-tools/scripts/local-top/cryptroot, the script that runs at boot time to unlock the root device. Note when a keyscript is set, it is simply run and the output piped to luksOpen with no other checking. There is no way to signal an error (USB drive not present) or fall back to keyboard input. If the passphrase fails, the keyscript is run again in a loop, up to some number of times; however we are not told which iteration we are on. Also, we have no control over when the keyscript is run, so we can't be sure Linux has recognized the USB drive.
I addressed this with some hacks:
Poll on the USB drive and wait 3 seconds for it to appear. This works for me, but I would love to know a better way.
Create a dummy file /passphrase-from-usb-tried on first run to indicate that we have been run at least once.
If we have been run at least once, or the USB drive cannot be found, run the askpass program used by cryptroot for keyboard input.
The final script:
#!/bin/sh
set -e
if ! [ -e /passphrase-from-usb-tried ]; then
touch /passphrase-from-usb-tried
if ! [ -e "$CRYPTTAB_KEY" ]; then
echo "Waiting for USB stick to be recognized..." >&2
sleep 3
fi
if [ -e "$CRYPTTAB_KEY" ]; then
echo "Unlocking the disk $CRYPTTAB_SOURCE ($CRYPTTAB_NAME) from USB key" >&2
dd if="$CRYPTTAB_KEY" bs=1 skip=129498880 count=256 2>/dev/null
exit
else
echo "Can't find $CRYPTTAB_KEY; USB stick not present?" >&2
fi
fi
/lib/cryptsetup/askpass "Unlocking the disk $CRYPTTAB_SOURCE ($CRYPTTAB_NAME)\nEnter passphrase: "
Finally, we need to ensure that this script is available in the initramfs. Create /etc/initramfs-tools/hooks/passphrase-from-usb containing:
#!/bin/sh
PREREQ=""
prereqs() {
echo "$PREREQ"
}
case "$1" in
prereqs)
prereqs
exit 0
;;
esac
. "${CONFDIR}/initramfs.conf"
. /usr/share/initramfs-tools/hook-functions
copy_exec /bin/passphrase-from-usb /bin
The USB drivers were not present in my initramfs. (It appears they are by default in later versions of Debian.) I had to add them by adding to /etc/initramfs-tools/modules:
uhci_hcd
ehci_hcd
usb_storage
When all is done, update the initramfs:
update-initramfs -u
It would be ideal to me if I could simply have a small USB stick containing a passphrase that will
unlock the disk. Not only would that be handy for servers (where you could leave the USB stick in the
server - the goal is to be able to return broken harddisks without having to worry about confidential data), it would also be great for my laptop: Insert the USB stick when booting and remove it after
unlocking the cryptodisk.
I have now written a patch that will search the root dir of all devices for the file 'cryptkey.txt' and try decrypting
with each line as a key. If that fails: Revert to typing in the pass phrase.
It does mean the key cannot contain \n, but that would apply to any typed in key, too. The good part is that you can use the same USB disk to store the key for multiple machines: You do not need a separate USB disk for each. So if you have a USB drive in your physical key ring, you can use the same drive for all the machines you boot when being physically close.
You add the key with:
cryptsetup luksAddKey /dev/sda5
And then put the same key as a line in a file on the USB/MMC disk called 'cryptkey.txt'. The patch is here:
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=864647
If the USB drivers, MMC drivers or the filesystems are not present in your initramfs, you need to add them by adding to /etc/initramfs-tools/modules:
uhci_hcd
ehci_hcd
usb_storage
nls_utf8
nls_cp437
vfat
fat
sd_mod
mmc_block
tifm_sd
tifm_core
mmc_core
tifm_7xx1
sdhci
sdhci_pci
When all is done, update the initramfs:
update-initramfs -u
It can be found as patch and file at: https://gitlab.com/ole.tange/tangetools/tree/master/decrypt-root-with-usb
despite the great answer from #Andrew which works in previous versions. The solution actually is outdated and needs lots of tuning for ubuntu 18.04 and 19.10. So I want to share my research on this.
There are several catches about crypttab. The sepcs actually changed a lot from 14.04 to 18.04 and to 19.10. It starts to support more parameters for cryptsetup. For example keyfile-offset, keyfile-size, etc. Some of the options e.g. nobootwait are gone. Some parameters supported in other distro already but is not supported in ubuntu yet (for example very nice parameter keyfile-timeout. This can eliminate the entire keyscript since it will automatically fallback to keyboard input after the keyfile-timeout.)
The major pit-fall for crypttab on ubuntu is that it actually processed by 2 different processes. One is the traditionally initramfs and another is the modern systemd. Systemd is supposed to be more advanced and flexiable in many aspect. However, systemd has poor support for crypptab, there are many options such as keyscript just silently ignored. so I have no idea what is going on, until I spotted this post. Almost all the posts online about crypttab settings is for initramfs not for systemd. So we need to add initramfs to all the entries in crypttab to avoid problems.
I also discovered a nice way to debug our keyscript and crypttab without VM or repeatedly rebooting. It is cryptdisks_start. Before we actually propagate our changes to initramfs, we should always test it with this nice command. Otherwise, you have to end-up locked out from your system and can only recover it through chroot environment.
#andrew posted a nice way to use data hide in the raw area of a file system. However, I found it is very annoying when we want to automatically create partitions and dd the raw data to lots of usbkeys, we have to calculate the offset for all different file systems and different partition sizes. Moreover, if a user accidentally write onto the FS, there is some risk that the key got overritten. A raw partition without any FS on it makes more sense in this case. However raw partition does not have UUID which is not very useful for automatic unlocking. Thus, I would like introduce a way just use normal passphrase files on the usbkey filesystem. The major issue of passdev is it does not seek/stop during reading the file. Thus we cannot use the keyfile-offset and keyfile-size option when we want to fallback to keyboard input. Because cryptsetup will actually try to skip in the input content and if the content is shorter than keyfile-size, it raises an error. This also means if there is large offset, passdev can be very slow since it always read from beginning. However, there is no point to implement offset and keyfile size for a actual file on file system. I believe those are created for raw device.
The crypttab
luks-part UUID="<uuid>" /dev/disk/by-uuid/<keyfile FS uuid>:/<keyfile path relative to usbkey root>:<timeout in sec> luks,keyfile-offset=<seek to the key>,keyfile-size=<>,keyscript=/bin/passphrase-from-usbfs.sh,tries=<number of times to try>,initramfs
the keyscript passphrase-from-usbfs.sh utilized the /lib/cryptsetup/scripts/passdev which will wait the usb device and mount the fs then pipe out the file content. It supports the CRYPTTAB_KEY in format of /device-path/<keyfile FS uuid>:/<keyfile path relative to usbkey root>:<timeout in sec>.
#!/bin/sh
#all message need to echo to stderr, the stdout is used for passphrase
# TODO: we may need to do something about the plymouth
echo "CRYPTTAB_KEY=$CRYPTTAB_KEY" >&2
echo "CRYPTTAB_OPTION_keyfile_offset=$CRYPTTAB_OPTION_keyfile_offset" >&2
#set your offset and file size here if your system does not support those paramters
#CRYPTTAB_OPTION_keyfile_offset=
#CRYPTTAB_OPTION_keyfile_size=
echo "timeout=$CRYPTTAB_OPTION_keyfile_timeout" >&2
CRYPTTAB_OPTION_keyfile_timeout=10 # keyfile-timeout is not supported yet
pass=$(/lib/cryptsetup/scripts/passdev $CRYPTTAB_KEY)
rc=$?
if ! [ $rc -eq 0 ]; then
echo "Can't find $CRYPTTAB_KEY; USB stick not present?" >&2
/lib/cryptsetup/askpass "Unlocking the disk $CRYPTTAB_SOURCE ($CRYPTTAB_NAME) Enter passphrase: "
else
echo "successfully load passphrase." >&2
echo -n $pass
fi
The hook tell update-initramfs to copy our scripts.
#!/bin/sh
PREREQ=""
prereqs() {
echo "$PREREQ"
}
case "$1" in
prereqs)
prereqs
exit 0
;;
esac
. "${CONFDIR}/initramfs.conf"
. /usr/share/initramfs-tools/hook-functions
copy_exec /bin/passphrase-from-usbfs.sh
copy_exec /bin/passphrase-from-usb.sh
#when using passdev we need to hook additionaly FS and binary
copy_exec /lib/cryptsetup/scripts/passdev
manual_add_modules ext4 ext3 ext2 vfat btrfs reiserfs xfs jfs ntfs iso9660 udf
Finally I posted the updated version of passphrase-from-usb.sh which can use the new parameters in crypttab:
To accompany excellent answers above please see C routines you could use to write/generate and read raw block device key. The "readkey.c" extracts key of given size from block device and "writekey.c" can generate or write existing key to raw device. The "readkey.c" once compiled can be used in custom script to extract key of known size from raw block device like so:
readkey </path/to/device> <keysize>
To see usage for "writekey", after compiled run it with no flags.
To compile just use:
gcc readkey.c -o readkey
gcc writekey.c -o writekey
I tested both on Verbatim 16GB USB 2.0 USB flash drive with custom "keyscript=" in crypttab also published below. The idea for "crypto-usb.sh" is from "debian etch" cryptsetup guide.
crypto-usb.sh:
#!/bin/sh
echo ">>> Trying to get the key from agreed space <<<" >&2
modprobe usb-storage >/dev/null 2>&1
sleep 4
OPENED=0
disk="/sys/block/sdb"
boot_dir="/boot"
readkey="/boot/key/readkey"
echo ">>> Trying device: $disk <<<" >&2
F=$disk/dev
if [ 0`cat $disk/removable` -eq 1 -a -f $F ]; then
mkdir -p $boot_dir
mount /dev/sda1 $boot_dir -t ext2 >&2
echo ">>> Attempting key extraction <<<" >&2
if [ -f $readkey ]; then
# prints key array to the caller
$readkey /dev/sdb 4096
OPENED=1
fi
umount $boot_dir >&2
fi
if [ $OPENED -eq 0 ]; then
echo "!!! FAILED to find suitable key !!!" >&2
echo -n ">>> Try to enter your password: " >&2
read -s -r A
echo -n "$A"
else
echo ">>> Success loading key <<<" >&2
fi
When generating the key size of the key has to be provided, generated key is saved to ".tmpckey" file with file permissions 0600 for later use. When writing existing key, size is determined by measuring the existing key size. This looks like complex approach however once compiled with simple "gcc" it can provide easy way of manipulating the raw key content.
readkey.c:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
void main(int argc, char *argv[])
{
int blockSize = 512;
int keySize = 2048;
FILE *device;
if ( argc == 3
&& (sizeof(argv[1]) / sizeof(char)) > 1
&& (sizeof(argv[2]) / sizeof(char)) > 1
&& (atoi(argv[2]) % 512) == 0
) {
device = fopen(argv[1], "r");
if(device == NULL) {
printf("\nI got trouble opening the device %s\n", argv[1]);
exit(EXIT_FAILURE);
}
keySize = atoi(argv[2]);
}
else if ( argc == 2
&& (sizeof(argv[1]) / sizeof(char)) > 1
) {
device = fopen(argv[1], "r");
if(device == NULL) {
printf("\nI got trouble opening the device %s\n", argv[1]);
exit(EXIT_FAILURE);
}
}
else {
printf("\nUsage: \n");
printf("\nKey Size Provided: \n");
printf("\n\t\treadkey </path/to/device> <keysize> \n");
printf("\nDefault key size: %d\n", keySize);
printf("\n\t\treadkey </path/to/device>\n");
exit(1);
}
int count;
char *block;
/* Verify if key is multiple of blocks */
int numBlocks = 0;
if (keySize % 512 != 0) {
printf("\nSory but key size is not multiple of block size, try again. TA.\n");
exit(1);
}
/* Seek till the end to get disk size and position to start */
fseek(device, 0, SEEK_END);
/* Determine where is the end */
long endOfDisk = ftell(device);
/* Make sure we start again */
rewind(device); // Do I need it ???
/* Get the required amount minus block size */
long startFrom = endOfDisk - blockSize - keySize;
/* Allocate space for bloc */
block = calloc(keySize, sizeof(char));
/* Start reading from specified block */
fseek(device, startFrom, SEEK_SET);
fread(block, 1, keySize, device);
/* Do something with the data */
for(count = 0; count < keySize/*sizeof(block)*/; count++){
printf("%c", block[count]);
}
/* Close file */
fclose(device);
/* Make sure freed array is zeroed */
memset(block, 0, keySize);
free(block);
}
writekey.c:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char *argv[])
{
int blockSize = 512;
int keySize = 2048;
int count;
unsigned char *block;
/*
Thing to always remember that argv starts from 0 - the name of the program, and argc starts from 1 i.e. 1 is the name of the program.
*/
if ( argc == 3
&& strcmp(argv[1], "genwrite") != 0
&& (sizeof(argv[2]) / sizeof(char)) > 2
) {
char ch;
FILE *keyF;
keyF = fopen(argv[1], "r");
if (keyF == NULL) exit(EXIT_FAILURE);
/* Tell key Size */
fseek(keyF, 0, SEEK_END);
keySize = ftell(keyF);
rewind(keyF);
printf("\nKey Size: %d\n", keySize);
block = calloc(keySize, sizeof(char));
printf("\n-- Start Key --:\n");
for(count = 0; count < keySize/*sizeof(block)*/; count++){
char ch = fgetc(keyF);
block[count] = ch;
/*
Uncomment below to see your key on screen
*/
// printf("%c",ch);
}
printf("\n-- End Key --:\n");
fclose(keyF);
}
else if ( argc == 3
&& strcmp(argv[1], "genwrite") == 0
&& (sizeof(argv[2]) / sizeof(char)) > 2
)
{
printf("\n-- Attempting to create random key(ish --) of size: %d\n", keySize);
block = calloc(keySize, sizeof(char));
int count;
for(count = 0; count < keySize/*sizeof(block)*/; count++){
block[count] = (char) rand();
}
FILE *tmpfile;
tmpfile = fopen(".tmpckey", "w");
if(tmpfile == NULL) exit(EXIT_FAILURE);
fwrite(block, 1, keySize, tmpfile);
fclose(tmpfile);
chmod(".tmpckey", 0600);
}
else if ( argc == 4
&& strcmp(argv[1], "genwrite") == 0
&& (sizeof(argv[2]) / sizeof(char)) > 2
&& ((atoi(argv[3]) % 512) == 0)
)
{
keySize = atoi(argv[3]);
printf("\n-- Attempting to create random key(ish --) of size: %d\n", keySize);
block = calloc(keySize, sizeof(char));
int count;
for(count = 0; count < keySize/*sizeof(block)*/; count++){
block[count] = (char) rand();
}
FILE *tmpfile;
tmpfile = fopen(".tmpckey", "w");
if(tmpfile == NULL) exit(EXIT_FAILURE);
fwrite(block, 1, keySize, tmpfile);
fclose(tmpfile);
chmod(".tmpckey", 0600);
}
else {
printf("\n");
printf("################################################################################\n");
printf("# #\n");
printf("# Usage: #\n");
printf("# #\n");
printf("################################################################################\n");
printf("#> To write existing key to device: #\n");
printf("# #\n");
printf("# writekey </path/to/keyfile> </path/to/removable/sd*> #\n");
printf("# #\n");
printf("#> To generate and write pseudo random key, #\n");
printf("#> key will be saved to temporary file .tmpckey #\n");
printf("# #\n");
printf("# writekey genwrite </path/to/removable/sd*> <keysize in multiples of 512> #\n");
printf("# #\n");
printf("#> When keysize is not provided default size is set to %d. #\n", keySize);
printf("# #\n");
printf("################################################################################\n");
exit(1);
}
/*
Some printf debugging below, uncomment when needed to see what is going on.
*/
/*
printf("\nNumber of Args: %d\n", argc);
printf("\nCurrently block array contains: \n");
for(count = 0; count < keySize; count++){
printf("%c", block[count]);
}
printf("\n-- End block -- \n");
*/
/* Open Device itp... */
FILE *device = fopen(argv[2], "a");
if(device == NULL) exit(EXIT_FAILURE);
printf("\nDevice to write: %s\n", argv[2]);
fseek(device, 0, SEEK_END);
/* Determine where is the end */
long endOfDisk = ftell(device);
printf("\nDevice Size: %ld\n", endOfDisk);
/* Verify if key is multiple of blocks */
int numBlocks = 0;
if (keySize % 512 != 0 || endOfDisk < (blockSize + keySize) ) {
printf("\nSorry but key size is not multiple of block size or device you trying to write to is too small, try again. TA.\n");
fclose(device);
exit(1);
}
/* Make sure we start again */
rewind(device);
/* Get the required amount sunbstracting block size */
long startFrom = endOfDisk - blockSize - keySize;
/* Write some data to the disk */
printf("\nWriting data starting from: %ld\n", startFrom);
fseek(device, startFrom, SEEK_SET);
fwrite(block, 1, keySize, device);
printf("\nBlock Position after data write procedure : %ld\n", ftell(device));
/*
Below is just for convenience, to read what was written,
can aid in debugging hence left commented for later.
*/
/*
printf("\nAmount of Data written : %ld\n", ftell(device) - startFrom);
// Start reading from specified block
printf("\n>>>>>>>> DEBUGGING SECTION <<<<<<<<<\n");
rewind(device); //
fseek(device, startFrom, SEEK_SET);
printf("\nBlock Position before read attempted: %d\n", ftell(device));
printf("\nKey size: %d\n", keySize);
fread(block, 1, keySize, device);
// Do something with the data
printf("\nBlock Position startFrom: %ld\n", startFrom);
printf("\nBlock Position after read: %d\n", ftell(device));
printf("\n-- Buffer Read: --\n");
for(count = 0; count < keySize; count++){
printf("%c", block[count]);
}
printf("\n-- End block -- \n");
printf("\n-- -- \n");
printf("\n-- -- \n");
*/
/* Close file */
fclose(device);
/* Make sure freed array is zeroed */
memset(block, 0, keySize);
free(block);
/* Return success, might change it to be useful return not place holder */
return 0;
}
To verify key written to raw device is the same as the one in file(below will output nothing if keys are identical):
diff -B <(./readkey </path/to/device> 4096) <(cat .tmpckey)
Or for existing key generated using own means:
diff -B <(./readkey </path/to/device> <generated elsewhere key size>) <(cat </path/to/keyfile>)
Thank You
Here is a solution similar to the one by Andrew, but
using CRYPTTAB_TRIED described in the Debian crypttab man page to distinguish tries, and
calling the existing standard keyscript /lib/cryptsetup/scripts/passdev on the first try.
Create your keyfile or keypartition as usual for the passdev script.
Create the following file /usr/local/bin/key-from-usb and make it executable.
#!/bin/sh
set -e
if [ $CRYPTTAB_TRIED -ge 1 ]; then
/lib/cryptsetup/askpass "Second try to unlock $CRYPTTAB_SOURCE ($CRYPTTAB_NAME). Please enter passphrase: "
else
/lib/cryptsetup/scripts/passdev $CRYPTTAB_KEY
fi
In /etc/crypttab use the parameter keyscript=/usr/local/bin/key-from-usb.
Create /etc/initramfs-tools/hooks/key-from-usb with this content:
#!/bin/sh
PREREQ=""
prereqs() {
echo "$PREREQ"
}
case "$1" in
prereqs)
prereqs
exit 0
;;
esac
. "${CONFDIR}/initramfs.conf"
. /usr/share/initramfs-tools/hook-functions
manual_add_modules vfat
copy_exec /usr/lib/cryptsetup/scripts/passdev /usr/lib/cryptsetup/scripts/passdev
copy_exec /usr/local/bin/key-from-usb /usr/local/bin/key-from-usb
The first copy_exec line here is needed because passdev is not copied if it is not mentioned in crypttab. Similarly, manual_add_modules vfat will ensure that a vfat usb disk can still be used.
Hint: Use lsinitramfs /boot/initrd.img-... and diff/compare the results to check that the script and all its dependencies are included.

Linux non-blocking fifo (on demand logging)

I like to log a programs output 'on demand'. Eg. the output is logged to the terminal, but another process can hook on the current output at any time.
The classic way would be:
myprogram 2>&1 | tee /tmp/mylog
and on demand
tail /tmp/mylog
However, this would create a ever growing log file even if not used until the drive runs out of space. So my attempt was:
mkfifo /tmp/mylog
myprogram 2>&1 | tee /tmp/mylog
and on demand
cat /tmp/mylog
Now I can read /tmp/mylog at any time. However, any output blocks the program until the /tmp/mylog is read. I like the fifo to flush any incoming data not read back. How to do that?
Inspired by your question I've written a simple program that will let you do this:
$ myprogram 2>&1 | ftee /tmp/mylog
It behaves similarly to tee but clones the stdin to stdout and to a named pipe (a requirement for now) without blocking. This means that if you want to log this way it may happen that you're gonna lose your log data, but I guess it's acceptable in your scenario.
The trick is to block SIGPIPE signal and to ignore error on writing to a broken fifo. This sample may be optimized in various ways of course, but so far, it does the job I guess.
/* ftee - clone stdin to stdout and to a named pipe
(c) racic#stackoverflow
WTFPL Licence */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
int main(int argc, char *argv[])
{
int readfd, writefd;
struct stat status;
char *fifonam;
char buffer[BUFSIZ];
ssize_t bytes;
signal(SIGPIPE, SIG_IGN);
if(2!=argc)
{
printf("Usage:\n someprog 2>&1 | %s FIFO\n FIFO - path to a"
" named pipe, required argument\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
readfd = open(fifonam, O_RDONLY | O_NONBLOCK);
if(-1==readfd)
{
perror("ftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status))
{
perror("ftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode))
{
printf("ftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK);
if(-1==writefd)
{
perror("ftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd);
while(1)
{
bytes = read(STDIN_FILENO, buffer, sizeof(buffer));
if (bytes < 0 && errno == EINTR)
continue;
if (bytes <= 0)
break;
bytes = write(STDOUT_FILENO, buffer, bytes);
if(-1==bytes)
perror("ftee: writing to stdout");
bytes = write(writefd, buffer, bytes);
if(-1==bytes);//Ignoring the errors
}
close(writefd);
return(0);
}
You can compile it with this standard command:
$ gcc ftee.c -o ftee
You can quickly verify it by running e.g.:
$ ping www.google.com | ftee /tmp/mylog
$ cat /tmp/mylog
Also note - this is no multiplexer. You can only have one process doing $ cat /tmp/mylog at a time.
This is a (very) old thread, but I've run into a similar problem of late. In fact, what I needed is a cloning of stdin to stdout with a copy to a pipe that is non blocking. the proposed ftee in the first answer really helped there, but was (for my use case) too volatile. Meaning I lost data I could have processed if I had gotten to it in time.
The scenario I was faced with is that I have a process (some_process) that aggregates some data and writes its results every three seconds to stdout. The (simplified) setup looked like this (in the real setup I am using a named pipe):
some_process | ftee >(onlineAnalysis.pl > results) | gzip > raw_data.gz
Now, raw_data.gz has to be compressed and has to be complete. ftee does this job very well. But the pipe I am using in the middle was too slow to grab the data flushed out - but it was fast enough to process everything if it could get to it, which was tested with a normal tee. However, a normal tee blocks if anything happens to the unnamed pipe, and as I want to be able to hook in on demand, tee is not an option. Back to the topic: It got better when I put a buffer in between, resulting in:
some_process | ftee >(mbuffer -m 32M| onlineAnalysis.pl > results) | gzip > raw_data.gz
But that was still losing data I could have processed. So I went ahead and extended the ftee proposed before to a buffered version (bftee). It still has all the same properties, but uses an (inefficient ?) internal buffer in case a write fails. It still loses data if the buffer runs full, but it works beautifully for my case. As always there is a lot of room for improvement, but as I copied the code off of here I'd like to share it back to people that might have a use for it.
/* bftee - clone stdin to stdout and to a buffered, non-blocking pipe
(c) racic#stackoverflow
(c) fabraxias#stackoverflow
WTFPL Licence */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
// the number of sBuffers that are being held at a maximum
#define BUFFER_SIZE 4096
#define BLOCK_SIZE 2048
typedef struct {
char data[BLOCK_SIZE];
int bytes;
} sBuffer;
typedef struct {
sBuffer *data; //array of buffers
int bufferSize; // number of buffer in data
int start; // index of the current start buffer
int end; // index of the current end buffer
int active; // number of active buffer (currently in use)
int maxUse; // maximum number of buffers ever used
int drops; // number of discarded buffer due to overflow
int sWrites; // number of buffer written to stdout
int pWrites; // number of buffers written to pipe
} sQueue;
void InitQueue(sQueue*, int); // initialized the Queue
void PushToQueue(sQueue*, sBuffer*, int); // pushes a buffer into Queue at the end
sBuffer *RetrieveFromQueue(sQueue*); // returns the first entry of the buffer and removes it or NULL is buffer is empty
sBuffer *PeakAtQueue(sQueue*); // returns the first entry of the buffer but does not remove it. Returns NULL on an empty buffer
void ShrinkInQueue(sQueue *queue, int); // shrinks the first entry of the buffer by n-bytes. Buffer is removed if it is empty
void DelFromQueue(sQueue *queue); // removes the first entry of the queue
static void sigUSR1(int); // signal handled for SUGUSR1 - used for stats output to stderr
static void sigINT(int); // signla handler for SIGKILL/SIGTERM - allows for a graceful stop ?
sQueue queue; // Buffer storing the overflow
volatile int quit; // for quiting the main loop
int main(int argc, char *argv[])
{
int readfd, writefd;
struct stat status;
char *fifonam;
sBuffer buffer;
ssize_t bytes;
int bufferSize = BUFFER_SIZE;
signal(SIGPIPE, SIG_IGN);
signal(SIGUSR1, sigUSR1);
signal(SIGTERM, sigINT);
signal(SIGINT, sigINT);
/** Handle commandline args and open the pipe for non blocking writing **/
if(argc < 2 || argc > 3)
{
printf("Usage:\n someprog 2>&1 | %s FIFO [BufferSize]\n"
"FIFO - path to a named pipe, required argument\n"
"BufferSize - temporary Internal buffer size in case write to FIFO fails\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
if (argc == 3) {
bufferSize = atoi(argv[2]);
if (bufferSize == 0) bufferSize = BUFFER_SIZE;
}
readfd = open(fifonam, O_RDONLY | O_NONBLOCK);
if(-1==readfd)
{
perror("bftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status))
{
perror("bftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode))
{
printf("bftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK);
if(-1==writefd)
{
perror("bftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd);
InitQueue(&queue, bufferSize);
quit = 0;
while(!quit)
{
// read from STDIN
bytes = read(STDIN_FILENO, buffer.data, sizeof(buffer.data));
// if read failed due to interrupt, then retry, otherwise STDIN has closed and we should stop reading
if (bytes < 0 && errno == EINTR) continue;
if (bytes <= 0) break;
// save the number if read bytes in the current buffer to be processed
buffer.bytes = bytes;
// this is a blocking write. As long as buffer is smaller than 4096 Bytes, the write is atomic to a pipe in Linux
// thus, this cannot be interrupted. however, to be save this should handle the error cases of partial or interrupted write none the less.
bytes = write(STDOUT_FILENO, buffer.data, buffer.bytes);
queue.sWrites++;
if(-1==bytes) {
perror("ftee: writing to stdout");
break;
}
sBuffer *tmpBuffer = NULL;
// if the queue is empty (tmpBuffer gets set to NULL) the this does nothing - otherwise it tries to write
// the buffered data to the pipe. This continues until the Buffer is empty or the write fails.
// NOTE: bytes cannot be -1 (that would have failed just before) when the loop is entered.
while ((bytes != -1) && (tmpBuffer = PeakAtQueue(&queue)) != NULL) {
// write the oldest buffer to the pipe
bytes = write(writefd, tmpBuffer->data, tmpBuffer->bytes);
// the written bytes are equal to the buffer size, the write is successful - remove the buffer and continue
if (bytes == tmpBuffer->bytes) {
DelFromQueue(&queue);
queue.pWrites++;
} else if (bytes > 0) {
// on a positive bytes value there was a partial write. we shrink the current buffer
// and handle this as a write failure
ShrinkInQueue(&queue, bytes);
bytes = -1;
}
}
// There are several cases here:
// 1.) The Queue is empty -> bytes is still set from the write to STDOUT. in this case, we try to write the read data directly to the pipe
// 2.) The Queue was not empty but is now -> bytes is set from the last write (which was successful) and is bigger 0. also try to write the data
// 3.) The Queue was not empty and still is not -> there was a write error before (even partial), and bytes is -1. Thus this line is skipped.
if (bytes != -1) bytes = write(writefd, buffer.data, buffer.bytes);
// again, there are several cases what can happen here
// 1.) the write before was successful -> in this case bytes is equal to buffer.bytes and nothing happens
// 2.) the write just before is partial or failed all together - bytes is either -1 or smaller than buffer.bytes -> add the remaining data to the queue
// 3.) the write before did not happen as the buffer flush already had an error. In this case bytes is -1 -> add the remaining data to the queue
if (bytes != buffer.bytes)
PushToQueue(&queue, &buffer, bytes);
else
queue.pWrites++;
}
// once we are done with STDIN, try to flush the buffer to the named pipe
if (queue.active > 0) {
//set output buffer to block - here we wait until we can write everything to the named pipe
// --> this does not seem to work - just in case there is a busy loop that waits for buffer flush aswell.
int saved_flags = fcntl(writefd, F_GETFL);
int new_flags = saved_flags & ~O_NONBLOCK;
int res = fcntl(writefd, F_SETFL, new_flags);
sBuffer *tmpBuffer = NULL;
//TODO: this does not handle partial writes yet
while ((tmpBuffer = PeakAtQueue(&queue)) != NULL) {
int bytes = write(writefd, tmpBuffer->data, tmpBuffer->bytes);
if (bytes != -1) DelFromQueue(&queue);
}
}
close(writefd);
}
/** init a given Queue **/
void InitQueue (sQueue *queue, int bufferSize) {
queue->data = calloc(bufferSize, sizeof(sBuffer));
queue->bufferSize = bufferSize;
queue->start = 0;
queue->end = 0;
queue->active = 0;
queue->maxUse = 0;
queue->drops = 0;
queue->sWrites = 0;
queue->pWrites = 0;
}
/** push a buffer into the Queue**/
void PushToQueue(sQueue *queue, sBuffer *p, int offset)
{
if (offset < 0) offset = 0; // offset cannot be smaller than 0 - if that is the case, we were given an error code. Set it to 0 instead
if (offset == p->bytes) return; // in this case there are 0 bytes to add to the queue. Nothing to write
// this should never happen - offset cannot be bigger than the buffer itself. Panic action
if (offset > p->bytes) {perror("got more bytes to buffer than we read\n"); exit(EXIT_FAILURE);}
// debug output on a partial write. TODO: remove this line
// if (offset > 0 ) fprintf(stderr, "partial write to buffer\n");
// copy the data from the buffer into the queue and remember its size
memcpy(queue->data[queue->end].data, p->data + offset , p->bytes-offset);
queue->data[queue->end].bytes = p->bytes - offset;
// move the buffer forward
queue->end = (queue->end + 1) % queue->bufferSize;
// there is still space in the buffer
if (queue->active < queue->bufferSize)
{
queue->active++;
if (queue->active > queue->maxUse) queue->maxUse = queue->active;
} else {
// Overwriting the oldest. Move start to next-oldest
queue->start = (queue->start + 1) % queue->bufferSize;
queue->drops++;
}
}
/** return the oldest entry in the Queue and remove it or return NULL in case the Queue is empty **/
sBuffer *RetrieveFromQueue(sQueue *queue)
{
if (!queue->active) { return NULL; }
queue->start = (queue->start + 1) % queue->bufferSize;
queue->active--;
return &(queue->data[queue->start]);
}
/** return the oldest entry in the Queue or NULL if the Queue is empty. Does not remove the entry **/
sBuffer *PeakAtQueue(sQueue *queue)
{
if (!queue->active) { return NULL; }
return &(queue->data[queue->start]);
}
/*** Shrinks the oldest entry i the Queue by bytes. Removes the entry if buffer of the oldest entry runs empty*/
void ShrinkInQueue(sQueue *queue, int bytes) {
// cannot remove negative amount of bytes - this is an error case. Ignore it
if (bytes <= 0) return;
// remove the entry if the offset is equal to the buffer size
if (queue->data[queue->start].bytes == bytes) {
DelFromQueue(queue);
return;
};
// this is a partial delete
if (queue->data[queue->start].bytes > bytes) {
//shift the memory by the offset
memmove(queue->data[queue->start].data, queue->data[queue->start].data + bytes, queue->data[queue->start].bytes - bytes);
queue->data[queue->start].bytes = queue->data[queue->start].bytes - bytes;
return;
}
// panic is the are to remove more than we have the buffer
if (queue->data[queue->start].bytes < bytes) {
perror("we wrote more than we had - this should never happen\n");
exit(EXIT_FAILURE);
return;
}
}
/** delete the oldest entry from the queue. Do nothing if the Queue is empty **/
void DelFromQueue(sQueue *queue)
{
if (queue->active > 0) {
queue->start = (queue->start + 1) % queue->bufferSize;
queue->active--;
}
}
/** Stats output on SIGUSR1 **/
static void sigUSR1(int signo) {
fprintf(stderr, "Buffer use: %i (%i/%i), STDOUT: %i PIPE: %i:%i\n", queue.active, queue.maxUse, queue.bufferSize, queue.sWrites, queue.pWrites, queue.drops);
}
/** handle signal for terminating **/
static void sigINT(int signo) {
quit++;
if (quit > 1) exit(EXIT_FAILURE);
}
This version takes one more (optional) argument which specifies the number of the blocks that are to buffered for the pipe. My sample call now looks like this:
some_process | bftee >(onlineAnalysis.pl > results) 16384 | gzip > raw_data.gz
resulting in 16384 blocks to be buffered before discards happen. this uses about 32 Mbyte more memory, but... who cares ?
Of course, in the real environment I am using a named pipe so that I can attach and detach as needed. There is looks like this:
mkfifo named_pipe
some_process | bftee named_pipe 16384 | gzip > raw_data.gz &
cat named_pipe | onlineAnalysis.pl > results
Also, the process reacts on signals as follows:
SIGUSR1 -> print counters to STDERR
SIGTERM, SIGINT -> first exits the main loop and flushed the buffer to the pipe, the second terminated the program immediatly.
Maybe this helps someone in the future...
Enjoy
However, this would create a ever growing log file even if not used until the drive runs out of space.
Why not periodically rotate the logs? There's even a program to do it for you logrotate.
There's also a system for generating log messages and doing different things with them according to type. It's called syslog.
You could even combine the two. Have your program generate syslog messages, configure syslog to place them in a file and use logrotate to ensure they don't fill the disk.
If it turned out that you were writing for a small embedded system and the program's output is heavy there are a variety of techniques you might consider.
Remote syslog: send the syslog messages to a syslog server on the network.
Use the severity levels availble in syslog to do different things with the messages. E.g. discard "INFO" but log and forward "ERR" or greater. E.g. to console
Use a signal handler in your program to reread configuration on HUP and vary log generation "on demand" this way.
Have your program listen on a unix socket and write messages down it when open. You could even implement and interactive console into your program this way.
Using a configuration file, provide granular control of logging output.
It seems like bash <> redirection operator (3.6.10 Opening File Descriptors for Reading and WritingSee) makes writing to file/fifo opened with it non-blocking.
This should work:
$ mkfifo /tmp/mylog
$ exec 4<>/tmp/mylog
$ myprogram 2>&1 | tee >&4
$ cat /tmp/mylog # on demend
Solution given by gniourf_gniourf on #bash IRC channel.
BusyBox often used on embedded devices can create a ram buffered log by
syslogd -C
which can be filled by
logger
and read by
logread
Works quite well, but only provides one global log.
The logging could be directed to a UDP socket. Since UDP is connection-less, it won't block the sending program. Of course logs will be lost if the receiver or network can't keep up.
myprogram 2>&1 | socat - udp-datagram:localhost:3333
Then when you want to observe the logging:
socat udp-recv:3333 -
There are some other cool benefits like being able to attach multiple listeners at the same time or broadcast to multiple devices.
If you can install screen on the embedded device then you can run 'myprogram' in it and detach it, and reattach it anytime you want to see the log. Something like:
$ screen -t sometitle myprogram
Hit Ctrl+A, then d to detach it.
Whenever you want to see the output, reattach it:
$ screen -DR sometitle
Hit Ctrl-A, then d to detach it again.
This way you won't have to worry about the program output using disk space at all.
The problem with the given fifo approach is that the whole thing will hang when the pipe buffer is getting filled up and no reading process is taking place.
For the fifo approach to work I think you would have to implement a named pipe client-server model similar to the one mentioned in BASH: Best architecture for reading from two input streams (see slightly modified code below, sample code 2).
For a workaround you could also use a while ... read construct instead of teeing stdout to a named pipe by implementing a counting mechanism inside the while ... read loop that will overwrite the log file periodically by a specified number of lines. This would prevent an ever growing log file (sample code 1).
# sample code 1
# terminal window 1
rm -f /tmp/mylog
touch /tmp/mylog
while sleep 2; do date '+%Y-%m-%d_%H.%M.%S'; done 2>&1 | while IFS="" read -r line; do
lno=$((lno+1))
#echo $lno
array[${lno}]="${line}"
if [[ $lno -eq 10 ]]; then
lno=$((lno+1))
array[${lno}]="-------------"
printf '%s\n' "${array[#]}" > /tmp/mylog
unset lno array
fi
printf '%s\n' "${line}"
done
# terminal window 2
tail -f /tmp/mylog
#------------------------
# sample code 2
# code taken from:
# https://stackoverflow.com/questions/6702474/bash-best-architecture-for-reading-from-two-input-streams
# terminal window 1
# server
(
rm -f /tmp/to /tmp/from
mkfifo /tmp/to /tmp/from
while true; do
while IFS="" read -r -d $'\n' line; do
printf '%s\n' "${line}"
done </tmp/to >/tmp/from &
bgpid=$!
exec 3>/tmp/to
exec 4</tmp/from
trap "kill -TERM $bgpid; exit" 0 1 2 3 13 15
wait "$bgpid"
echo "restarting..."
done
) &
serverpid=$!
#kill -TERM $serverpid
# client
(
exec 3>/tmp/to;
exec 4</tmp/from;
while IFS="" read -r -d $'\n' <&4 line; do
if [[ "${line:0:1}" == $'\177' ]]; then
printf 'line from stdin: %s\n' "${line:1}" > /dev/null
else
printf 'line from fifo: %s\n' "$line" > /dev/null
fi
done &
trap "kill -TERM $"'!; exit' 1 2 3 13 15
while IFS="" read -r -d $'\n' line; do
# can we make it atomic?
# sleep 0.5
# dd if=/tmp/to iflag=nonblock of=/dev/null # flush fifo
printf '\177%s\n' "${line}"
done >&3
) &
# kill -TERM $!
# terminal window 2
# tests
echo hello > /tmp/to
yes 1 | nl > /tmp/to
yes 1 | nl | tee /tmp/to
while sleep 2; do date '+%Y-%m-%d_%H.%M.%S'; done 2>&1 | tee -a /tmp/to
# terminal window 3
cat /tmp/to | head -n 10
If your process writes to any log file and then wipes the file and starts again every now and again, so it doesn't get too big, or uses logrotate.
tail --follow=name --retry my.log
Is all you need. You will get as much scroll-back as your terminal.
Nothing non standard is needed. I've not tried it with small log files but all our logs rotate like this and I have never noticed loosing lines.
To follow in Fabraxias foot steps I'm going to share my small modification of racic's code. In one of my use cases I needed to suppress the writes to STDOUT, so I've added another parameter: swallow_stdout. If that is not 0, then output to STDOUT will be turned off.
Since I'm no C coder I've added comments while reading the code, maybe they are useful for others.
/* ftee - clone stdin to stdout and to a named pipe
(c) racic#stackoverflow
WTFPL Licence */
// gcc /tmp/ftee.c -o /usr/local/bin/ftee
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <signal.h>
#include <unistd.h>
int main(int argc, char *argv[])
{
int readfd, writefd; // read & write file descriptors
struct stat status; // read file descriptor status
char *fifonam; // name of the pipe
int swallow_stdout; // 0 = write to STDOUT
char buffer[BUFSIZ]; // read/write buffer
ssize_t bytes; // bytes read/written
signal(SIGPIPE, SIG_IGN);
if(3!=argc)
{
printf("Usage:\n someprog 2>&1 | %s [FIFO] [swallow_stdout] \n"
"FIFO - path to a named pipe (created beforehand with mkfifo), required argument\n"
"swallow_stdout - 0 = output to PIPE and STDOUT, 1 = output to PIPE only, required argument\n", argv[0]);
exit(EXIT_FAILURE);
}
fifonam = argv[1];
swallow_stdout = atoi(argv[2]);
readfd = open(fifonam, O_RDONLY | O_NONBLOCK); // open read file descriptor in non-blocking mode
if(-1==readfd) // read descriptor error!
{
perror("ftee: readfd: open()");
exit(EXIT_FAILURE);
}
if(-1==fstat(readfd, &status)) // read descriptor status error! (?)
{
perror("ftee: fstat");
close(readfd);
exit(EXIT_FAILURE);
}
if(!S_ISFIFO(status.st_mode)) // read descriptor is not a FIFO error!
{
printf("ftee: %s in not a fifo!\n", fifonam);
close(readfd);
exit(EXIT_FAILURE);
}
writefd = open(fifonam, O_WRONLY | O_NONBLOCK); // open write file descriptor non-blocking
if(-1==writefd) // write file descriptor error!
{
perror("ftee: writefd: open()");
close(readfd);
exit(EXIT_FAILURE);
}
close(readfd); // reading complete, close read file descriptor
while(1) // infinite loop
{
bytes = read(STDIN_FILENO, buffer, sizeof(buffer)); // read STDIN into buffer
if (bytes < 0 && errno == EINTR)
continue; // skip over errors
if (bytes <= 0)
break; // no more data coming in or uncaught error, let's quit since we can't write anything
if (swallow_stdout == 0)
bytes = write(STDOUT_FILENO, buffer, bytes); // write buffer to STDOUT
if(-1==bytes) // write error!
perror("ftee: writing to stdout");
bytes = write(writefd, buffer, bytes); // write a copy of the buffer to the write file descriptor
if(-1==bytes);// ignore errors
}
close(writefd); // close write file descriptor
return(0); // return exit code 0
}

Resources