Perl - multithreading / fork / synchronize problem: - multithreading

I'm trying to work as an example for the following code:
my $milon;
my $pid = fork();
die if not defined $pid;
if (not $pid)
{
$milon->{$pid} = $pid;
exit;
}
$milon->{3} = 4;
my $finished = wait();
print( "debug10: TEST = ", Dumper($milon));
output:
debug10: TEST = $VAR1 = {
'3' => 4
};
How do I make the dictionary keep both 3 => 4 and also the $pid => $pid?
It doesn't have to be forking, it could be multithreading or NonBlocking IO, whichever is better according to what you think.
This is an example of course, I just want to conclude from this example to my real code.

You need some memory that is shared between your threads/processes. The easiest is probably to use interpreter-based threads and threads::shared. For instance:
use threads;
use threads::shared;
my %milon :shared;
for (1 .. 2) {
threads->create(sub {
my $tid = threads->tid();
$milon{$tid} = $tid;
});
}
$milon{3} = 4;
$_->join for threads->list; # Wait for all threads to be done
print Dumper \%milon;
This outputs:
$VAR1 = {
'1' => 1,
'2' => 2,
'3' => 4
};

Following sample code demonstrates usage of fork() to compute square of a number from an array #numbers (total 100) in parallel execution.
REAPER function assigned to $SIG{CHLD} signal cleans up completed child processes to avoid zombie processes hanging around in process table.
Investigate if fork() approach will fit your problem/task.
use strict;
use warnings;
use POSIX qw(strftime :sys_wait_h);
use Time::HiRes qw(usleep);
my $limit = 10;
my $threads = $limit;
my #numbers = map { int(rand(100)) } 1..100;
sub REAPER {
local $!;
while( (my $pid = waitpid(-1, WNOHANG) ) > 0 && WIFEXITED($?) ) {
$threads++;
}
$SIG{CHLD} = \&REAPER;
}
$SIG{CHLD} = \&REAPER;
for ( #numbers ) {
while( $threads == 0 or $threads > $limit ) { usleep(1) }
my $pid = fork();
die $! unless defined $pid;
if( $pid ) {
# parent
$threads--;
} else {
# child
my $n = compute_square($_);
printf "Process %6d: %3d => %d\n", $$, $_, $n;
exit 0;
}
}
sub compute_square {
my $num = shift;
return $num*$num;
}

Related

Perl: Make thread sleep

I have a basic Perl script that runs a function with 20 threads.
use threads;
use threads::shared;
use Thread::Queue;
use File::Slurp;
$| = 1; my $numthreads = 20;
my $fetch_q = Thread::Queue->new();
sub fetch {
while ( my $target = $fetch_q->dequeue() ) {
my $request = `curl "http://WEBSITE" -s -o /dev/null -w "%{http_code}"`; # Returns HTTP Status code of request (i.e. 200, 302, 404, etc)
if ($request eq "200") {
print "Success. Sleeping for 5 seconds.";
sleep(5);
}
else {
print "Fail. Will try again in 10 seconds.";
sleep(10);
redo;
}
}
}
my #workers = map { threads->create( \&fetch ) } 1 .. $numthreads;
$fetch_q->enqueue( 1 .. $max );
$fetch_q->end();
foreach my $thr (#workers) {$thr->join();}
If a condition is true, I want the thread to sleep for 5 seconds but have all the other threads continue. Right now when I use sleep(5) it seems like the entire script sleeps for 5 seconds.
How can I use sleep() for individual threads?
sleep is the right tool. I don't know why you think it seems to block all threads, but it doesn't. The following illustrates this:
use threads;
use threads::shared;
use Thread::Queue;
my $fetch_q = Thread::Queue->new();
sub fetch {
while (defined( my $job = $fetch_q->dequeue() )) {
if ($job) {
print threads->tid, " Starting to sleep\n";
sleep(5);
print threads->tid, " Finished sleeping\n";
}
else {
print threads->tid, " Starting to sleep\n";
sleep(10);
print threads->tid, " Finished sleeping\n";
}
}
}
my #workers = map { threads->create( \&fetch ) } 1 .. 2;
$fetch_q->enqueue($_) for 0, 1, 1;
$fetch_q->end();
foreach my $thr (#workers) {$thr->join();}
Output:
1 Starting to sleep
2 Starting to sleep
2 Finished sleeping
2 Starting to sleep <-- The second job picked up work while the first was still sleeping.
1 Finished sleeping
2 Finished sleeping

Exit perl script automatically every 2 hours

I have a perl scipt that I need it to end every two hours on a Linux machine. I was going to do a separate script and add it on cron.d to accomplish this, but I wanted an easier way. It has to do a graceful exit because after doing a CTRL+C, it writes a log file and killing it won't write the file.
You can set up an alarm at the beginning of the script, and provide a handler for it:
alarm 60 * 60 * 2;
local $SIG{ALRM} = sub {
warn "Time over!\n";
# Do the logging here...
exit
};
The question is how you would restart the script again.
A wrapper keeps things simple.
#!/usr/bin/perl
# usage:
# restarter program [arg [...]]
use strict;
use warnings;
use IPC::Open3 qw( open3 );
use POSIX qw( WNOHANG );
use constant RESTART_AFTER => 2*60*60;
use constant KILL_INT_WAIT => 30;
use constant KILL_TERM_WAIT => 30;
use constant WAIT_POLL => 15;
sub start_it {
open(local *NULL, '<', '/dev/null')
or die($!);
return open3('<&NULL', '>&STDOUT', '>&STDERR', #_);
}
sub wait_for_it {
my ($pid, $max_wait) = #_;
my $end_time = time + $max_wait;
while (1) {
if (waitpid($pid, WNOHANG) > 0) {
return 1;
}
my $time = time;
if ($end_time >= $time) {
return 0;
}
sleep(1);
}
}
sub end_it {
my ($pid) = #_;
kill(INT => $pid)
or die($!);
return if wait_for_it($pid, KILL_INT_WAIT);
kill(TERM => $pid)
or die($!);
return if wait_for_it($pid, KILL_TERM_WAIT);
kill(KILL => $pid)
or die($!);
waitpid($pid, 0);
}
sub run_it {
my $end_time = time + RESTART_AFTER;
my $pid = start_it(#_);
while (1) {
if (waitpid($pid, WNOHANG) > 0) {
last;
}
my $time = time;
if ($end_time >= $time) {
end_it($pid);
last;
}
my $sleep_time = $end_time - $time;
$sleep_time = WAIT_POLL if $sleep_time > WAIT_POLL; # Workaround for race condition.
sleep($sleep_time);
}
my $status = $?;
if ($? & 0x7F) { warn("Child killed by signal ".($? & 0x7F)."\n"); }
elsif ($? >> 8) { warn("Child exited with error ".($? >> 8)."\n"); }
else { warn("Child exited with succcesfully.\n"); }
}
run_it(#ARGV) while 1;
You might want to forward signals sent to the handler to the child.
You can catch the signal sent by Ctrl-C by setting a subroutine in $SIG{INT}:
$ perl -e '$SIG{INT} = sub { print "Caught signal, cleaning up\n"; exit 0 }; while(1) {}'
Do your cleanup within the sub, and there you go.

How to pause and resume a multithread perl script?

I have written the perl script to pause and resume.When the user enters Ctrl+c it has to pause and on pressing c it should resume. But is not working properly as expected. Can anyone help me on this what mistake i am making:
use strict;
use threads;
use threads::shared;
use Thread::Suspend;
use Lens;
$SIG{'INT'} = 'Pause';
#$| = 1;
print chr(7);
my $nthreads = 64;
my #thrs;
for(1..$nthreads)
{
print "START $_ \n";
my ($thr) = threads->create(\&worker, $_);
push #thrs ,$thr;
}
$_->join for #thrs;
exit;
sub worker
{
my $id = shift;
my $tmp;
my $lens = Lens->new("172.16.1.65:2000");
die "cannot create object" unless defined $lens;
die "cannot connect to XRay at " unless defined $lens->open("172.16.1.65:2000");
for(1..100000)
{
print "Thread $id \n";
}
print "$id>LOAD EXIT\n";
}
sub Pause
{
sleep(1);
print "\nCaught ^C\n";
print "Press \"c\" to continue, \"e\" to exit: ";
$_->suspend() for #thrs;
while (1)
{
my $input = lc(getc());
chomp ($input);
if ($input eq 'c') {
#clock($hour,$min,$sec);
$_->resume() for #thrs;
return;
}
elsif ($input eq 'e') {
exit 1;
}
}
}
Well, you haven't been too specific as to how it's "not working properly". But I would suggest looking at using Thread::Semaphore for a 'suspend' mechanism.
I would also suggest not using signal and instead doing something like:
#!/usr/bin/perl
use strict;
use warnings;
use threads;
use Thread::Semaphore;
use Term::ReadKey;
my $nthreads = 64;
my $thread_semaphore = Thread::Semaphore->new($nthreads);
sub worker {
for ( 1 .. 10 ) {
$thread_semaphore->down();
print threads->self->tid(), "\n";
sleep 1;
$thread_semaphore->up();
}
}
for ( 1 .. $nthreads ) {
threads->create( \&worker );
}
my $keypress;
ReadMode 4;
while ( threads->list(threads::running) ) {
while ( not defined( $keypress = ReadKey(-1) )
and threads->list(threads::running) )
{
print "Waiting\nRunning:". threads->list(threads::running) . "\n";
sleep 1;
}
print "Got $keypress\n";
if ( $keypress eq "p" ) {
print "Pausing...";
$thread_semaphore -> down_force($nthreads);
print "All paused\n";
}
if ( $keypress eq "c" ) {
print "Resuming...";
$thread_semaphore -> up ( $nthreads );
}
}
ReadMode 0;
foreach my $thr ( threads->list ) {
$thr->join();
}
It'll 'suspend' by setting the semaphores to zero (or negative) and relies on the threads checking if they should be stopping here or not.
I think the root of your problem though, will probably be signal propagation - your signal handler is global across your threads. You might find configuring $SIG{'INT'} for your threads separately will yield better results. (E.g. set the signal handler to 'IGNORE' at the start of your code, and set specific ones in the thread/main once the threads have been spawned).

In Perl, how can a child thread signal to the main thread that no more threads should be created?

I'm having some problems in returning a value from a thread in perl.
The code I'm using is this:
use threads;
foreach $num(1 .. 100)
{
push(#threads, threads->create (\&readnum, $num));
sleep(1) while(scalar threads->list(threads::running) >= 10);
}
$_->join foreach #threads;
sub readnum {
# some code here
}
so I want to return a value from readnum i.e:
use threads;
foreach $num(1 .. 100)
{
if($ok)
{
push(#threads, threads->create (\&readnum, $num));
sleep(1) while(scalar threads->list(threads::running) >= 10);
}
}
$_->join foreach #threads;
sub readnum {
# some code here
return $ok ? "1" : "0";
}
So I want to check the value of $ok if it's true it'll create a new thread.
edit:
what i want is to check for $ok value, if it's true it'll creat a new thread and keep progress else it stop. the same idea without threads :
foreach $num(1 .. 100)
{
$ok = readnum($num);
print "runing\n";
die "stoped\n" if $ok eq 1;
}
sub readnum {
# some code here
$_[0]/5 eq 2 ? return 1 : return 0;
}
but with thread i can't put the returned value in $ok.
hope it's clear now. thanks
What you are trying to do is not return a value from a thread, but have one thread the ability to signal to the main thread that no more threads should be created. You can do that by creating a shared global. In the example below, there is a 1/20 chance that a random thread will decide that processing should be stopped.
There may be other, better ways of dealing with your specific problem (for example, each thread pushing results to a shared array, and the main thread checking how many results are there etc), but this seems to match your situation.
#!/usr/bin/env perl
use strict;
use warnings;
use threads;
use threads::shared;
my $KEEP_GOING :shared;
$KEEP_GOING = 1;
my #threads;
THREAD:
for my $num (1 .. 100) {
sleep 1 while threads->list(threads::running) >= 10;
{
lock $KEEP_GOING;
if($KEEP_GOING) {
push #threads, threads->create(\&readnum, $num);
}
else {
print "Won't create thread for $num .. Goodbye!\n";
last THREAD;
}
}
}
$_->join for #threads;
sub readnum {
my $num = shift;
printf "Thread id: %d\tnum = %d\n", threads->tid, $num;
sleep 1 + rand(3);
{
lock $KEEP_GOING;
if (0.05 > rand) {
$KEEP_GOING = 0;
}
}
}
Output:
Thread id: 1 num = 1
Thread id: 2 num = 2
Thread id: 3 num = 3
Thread id: 4 num = 4
Thread id: 5 num = 5
Thread id: 6 num = 6
Thread id: 7 num = 7
Thread id: 8 num = 8
Thread id: 9 num = 9
Thread id: 10 num = 10
Won't create thread for 11 .. Goodbye!

Perl Multithread Program

I am new in Perl. I want to write a Perl script using thread.I have few files say 20 files and want to process those files using 5 threads in 4 batches. I am printing the thread no. After completing one batch ,the thread no must start with 1 for the next batch. But instead of that its creating 20 threads.please help. my code is as follows:
#!/usr/bin/perl -w
use strict;
use warnings;
use threads;
use threads::shared;
my $INPUT_DIR="/home/Documents/myscript/IMPORTLDIF/";
opendir(DIR, $INPUT_DIR) ;
my #files = grep { /^InputFile/ } readdir DIR;
my $count = #files;
#print "Total Files: $count \n";
my #threads;
my $noofthread = 5;
my $nooffiles = $count;
my $noofbatch = $nooffiles / $noofthread;
#print "No of batch: $noofbatch \n";
my $fileIndex = 0;
my $batch = 1;
while ($fileIndex < $nooffiles) {
print "Batch: $batch \n";
for (my $i=0; $i < $noofthread && $fileIndex < $nooffiles ; $i++) {
my $t = threads->new(\&doOperation, $files[$fileIndex], $i)->join;
push(#threads, $t);
$fileIndex++;
print "FileIndex: $fileIndex \n";
}
$batch++;
}
sub doOperation () {
my $ithread = threads->tid() ;
print "Thread Index : [id=$ithread]\n" ;
foreach my $item (#_){
my $filename = $item;
print "Filename name: $filename \n";
}
Edited program using thread queue:
#!/usr/bin/perl -w
# This is compiled with threading support
use strict;
use warnings;
use threads;
use Thread::Queue;
my $q = Thread::Queue->new(); # A new empty queue
# Worker thread
my $INPUT_DIR="/home/Documents/myscript/IMPORTLDIF/";
opendir(DIR, $INPUT_DIR) or die "Cannot opendir: $!";
my #thrs = threads->create(\&doOperation ) for 1..5;#for 5 threads
#my #files = `ls -1 /home/Documents/myscript/IMPORTLDIF/`;
my #files = grep { /^Input/ } readdir DIR or die "File not present present. \n";
chomp(#files);
#add files to queue
foreach my $f (#files){
# Send work to the thread
$q->enqueue($f);
print "Pending items: " + $q->pending()."\n";
}
$q->enqueue('_DONE_') for #thrs;
$_->join() for #thrs;
sub doOperation () {
my $ithread = threads->tid() ;
while (my $filename = $q->dequeue()) {
# Do work on $item
return 1 if $filename eq '_DONE_';
print "[id=$ithread]\t$filename\n";
}
return 1;
}
You are spawning a thread and then waiting for it to complete before spawning the next, each thread handling one file. That is why you see as many threads as you have files.
my $t = threads->new(\&doOperation, $files[$fileIndex], $i)->join;
^^^^--- This will block
Instead try something like this:
....
# split the workload into N batches
#
while (my #batch = splice(#files, 0, $batch_size)) {
push #threads, threads->new(\&doOperation, #batch);
}
# now wait for all workers to finish
#
for my $thr (#threads) {
$thr->join;
}
As an aside, Thread::Queue and Thread-Pool might imply better designs for the work you want to do.
You could use Paralel:Queue and create 4 thread and pass them items that they could work on.
To fork or not to fork?
use strict;
use warnings;
use threads;
use Thread::Queue;
my $q = Thread::Queue->new(); # A new empty queue
# Worker thread
my #thrs;
push #thrs, threads->create(\&doOperation ) for 1..5;#for 5 threads
my #files = `ls -1 /tmp/`;chomp(#files);
#add files to queue
foreach my $f (#files){
# Send work to the thread
$q->enqueue($f);
print "Pending items: "$q->pending()."\n";
}
$q->enqueue('_DONE_') for #thrs;
$_->join() for threads->list();
sub doOperation () {
my $ithread = threads->tid() ;
while (my $filename = $q->dequeue()) {
# Do work on $item
return 1 if $filename eq '_DONE_';
print "[id=$ithread]\t$filename\n";
}
return 1;
}

Resources