Send message when failed(Python) - python-3.x

Im trying to simulate two machines working, and failing at random times. When they fail they call assistance. These two machines is part of bigger system of different machines, which needs to know when its neighbor has failed to do its job.
So far, I have made the simulate of the two machines, but I cant figure out how to send messages to their neighbors without each machine needing to know the whole system?
This is what I have so far:
import simpy
import random
random_seed=42
MTTF = 3500
break_mean = 1 / MTTF
sim_time = 4 * 7*24*60 # 4 weeks 24/7
num_machines = 2
rep_time = 30
tpp = 20 #20 minutes to make each part
neighbour = 3 #How many should it send to?
#Creating a class called messaging which is an object.
class messaging(object):
#DEfing the initilizing function, and initilize self, Environment, and capacity which is set to infinity, by a simpy core-function.
def __init__(self, env, capacity=simpy.core.Infinity):
self.env = env
self.capacity = capacity
self.pipes = []
#Making a function which work on everything that is a part of the message. With name Put.
def put(self, value):
if not self.pipes: #How to get this error?
raise runtime_error('There are no output pipes.')
#Create a variable, events, store to it pipe values
events = broken_machine()
return self.env.all_of(events)
def get_output_conn(self):
#Set the capacity of pipe variable to store infinity.
pipe = simpy.Store(self.env, capacity=self.capacity)
#to each pipes, add(or append) pipe
self.pipes.append(pipe)
return pipe
def mesg_generator(number, env, out_pipe):
msg = ('Failed')
def message_reciever(name, env, in_pipe):
while True:
msg = yield in_pipe.get()
print("%s received message: %s" % (number, msg[1]))
def time_per_part():
return tpp
def ttf():
return random.expovariate(break_mean)
class Machine(object):
def __init__(self, env, number, repair):
#self.arg = arg
self.env = env
self.number = number
self.parts_made = 0
self.times_broken = 0
self.broken = False
self.process = env.process(self.working(repair))
env.process(self.broken_machine())
def working(self, repair):
while True:
work = time_per_part()
while work:
try:
begin = self.env.now
yield self.env.timeout(work)
work = 0
except simpy.Interrupt:
self.broken = True
work -= self.env.now - begin
with repair.request(priority = 1) as req:
yield req
yield self.env.timeout(rep_time)
self.times_broken +=1
yield message_reciever()
#print('Machine down')
self.broken = False #Machine fixed again
self.parts_made +=1
def broken_machine(self):
while True:
yield self.env.timeout(ttf())
if not self.broken:
self.process.interrupt()
def other_jobs(env, repair):
while True:
work = tpp
while work:
with repair.request(priority=2) as req:
yield req
try:
begin = env.now
yield env.timeout(work)
work = 0
except simpy.Interrupt:
work -= env.now - begin
print("This simulates machines 3 and 4 doing the same tasks.")
random.seed(random_seed)
env = simpy.Environment()
pipe = simpy.Store(env)
bc_pipe = messaging(env)
repair = simpy.PreemptiveResource(env, capacity = 1)
machines = [Machine(env, 'Machine %d' % i, repair)
for i in range(num_machines)]
env.process(other_jobs(env, repair))
env.run(until=sim_time)
#Show how many times each machine failed:
for machine in machines:
print("%s broke down %d times" %(machine.number, machine.times_broken))

Related

PyInstaller creates an exe file, it works for the menu but it doesn't work for the actual program

Okay so I'm using pyinstaller to create an exe file for my python project to run, but when I run it, the exe file will open up my menu, and it will work perfectly fine, but when I select an option, it will clear the screen and then close the program. The program works perfectly well when run from command line. Below is the menu that I using pyinstaller on.
from pick import pick
import subprocess
def select_difficulty():
title = "Welcome to LowKeyBored's implementation of Typing Test (Rev 0)!"+'\nSelect difficulty'
options = ['Easy','Medium','Hard']
option, index = pick(options, title)
return option
def set_difficulty():
diff_args = ['-v']
difficulty = select_difficulty()
if difficulty == 'Easy':
diff_args.append('easy')
elif difficulty == 'Medium':
diff_args.append('medium')
else:
diff_args.append('hard')
return diff_args
def main():
diff_args = set_difficulty()
launch_args = ['python3','runner.py']+diff_args
subprocess.run(launch_args)
main()
runner .py
import typing_test
typing_test.main()
typingtest.py
"""
Copyright (c) 2019 Emil Lynegaard
Distributed under the MIT software license, see the
accompanying LICENSE.md or https://opensource.org/licenses/MIT
Minimal 10fastfingers like typing game for Python2/3 with ncurses.
Supports vocab files as arguments, as well as adjustable
word length, game time and word frequency (if using sorted vocab).
"""
import argparse
import time
import random
import curses
import textwrap
import os
# Robust path to default vocabulary, which is based on word frequency
# from CNN and DailyMail articles.
VOCAB_PATH = os.path.join(os.path.dirname(__file__), "data", "vocab")
# Used for WPM calculation
CHARS_PER_WORD = 5
# Amount of words to store, for showing several future words
# Should generally be equal to at least two lines worth of words
QUEUE_SIZE = 30
# Maximum amount of characters to be displayed in a single row
MAX_WIDTH = 80
# pylint: disable=too-many-instance-attributes
class Game:
"""
Class encapsulating the Game.
Includes game stats, input management, game display.
"""
CORRECT_COLOR = 1
INCORRECT_COLOR = 2
def __init__(self, args):
self.word_generator = self._word_generator(args)
self.game_time = args.game_time
self.next_words = [self._get_word() for _ in range(QUEUE_SIZE)]
self.typed = []
self.correct = []
self.incorrect = []
self.input = ""
self.display = args.display
# if using 10ff display, we keep track of extra things
if self.display == "10ff":
self.offset = 0
self.current_line = []
self.next_line = []
#staticmethod
def _word_generator(args):
words = []
for line in open(os.path.join(os.path.dirname(__file__), "data",args.vocab)):
word = line.strip()
if args.min_length <= len(word) <= args.max_length:
words.append(word)
if len(words) >= args.words:
break
if args.mode == "word":
while True:
yield random.choice(words)
else:
while True:
yield from words
def calculate_cpm(self, time_played):
"""Calculate CPM given time_played in seconds"""
if time_played == 0:
return 0
correct_chars = len(" ".join(self.correct))
cpm = 60 / time_played * correct_chars
cpm = int(round(cpm))
return cpm
def calculate_wpm(self, time_played):
"""Calculate WPM given time_played in seconds"""
cpm = self.calculate_cpm(time_played)
wpm = cpm // CHARS_PER_WORD
return wpm
def _get_word(self):
return next(self.word_generator)
def _finish_word_event(self):
target = self.next_words.pop(0)
self.typed.append(self.input)
if self.input == target:
self.correct.append(target)
else:
self.incorrect.append(target)
if self.display == "10ff":
self.offset += 1
self.next_words.append(self._get_word())
self.input = ""
#staticmethod
def _get_line(words, max_chars):
line = []
chars = 0
for word in words:
length = len(word)
# use +1 to account for added whitespace
if chars + length + 1 > max_chars:
break
line.append(word)
chars += length + 1
return line
def _progressive_display(self, stdscr, time_left):
_height, width = stdscr.getmaxyx()
width = min(width, MAX_WIDTH)
stdscr.clear()
wpm = self.calculate_wpm(self.game_time - time_left)
stdscr.addstr("Time left: {:d}, WPM: {:d}\n".format(time_left, wpm)+", Difficulty Level: "+self.difficulty_level+"\n")
line = self._get_line(self.next_words, width)
target = " ".join(line)
for idx, char in enumerate(self.input):
target_char = target[idx]
if target_char == char:
stdscr.addstr(char, curses.color_pair(self.CORRECT_COLOR))
else:
stdscr.addstr(target_char, curses.color_pair(self.INCORRECT_COLOR))
stdscr.addstr(target[len(self.input) : width - 1])
stdscr.addstr("\n" + self.input, curses.A_UNDERLINE)
stdscr.refresh()
def _10ff_display(self, stdscr, time_left):
_height, width = stdscr.getmaxyx()
width = min(width, MAX_WIDTH)
stdscr.clear()
wpm = self.calculate_wpm(self.game_time - time_left)
# stdscr.addstr("Time left: {:d}, WPM: {:d}".format(time_left, wpm)+", Difficulty Level: "+self.difficulty_level+"\n")
stdscr.addstr("Time left: {:d}, WPM: {:d}".format(time_left, wpm)+"\n")
# sets up initial lines
if not self.current_line:
self.current_line = self._get_line(self.next_words, width)
cur_len = len(self.current_line)
self.next_line = self._get_line(self.next_words[cur_len:], width)
# if we finished the current line
if self.offset >= len(self.current_line):
self.current_line = self.next_line
cur_len = len(self.current_line)
self.next_line = self._get_line(self.next_words[cur_len:], width)
self.offset = 0
# color the words already typed on current line
for i in range(self.offset):
target = self.current_line[i]
actual = self.typed[-(self.offset - i)]
if actual == target:
stdscr.addstr(target, curses.color_pair(self.CORRECT_COLOR))
else:
stdscr.addstr(target, curses.color_pair(self.INCORRECT_COLOR))
stdscr.addstr(" ")
stdscr.addstr(" ".join(self.current_line[self.offset :]))
stdscr.addstr("\n" + " ".join(self.next_line))
stdscr.addstr("\n" + self.input, curses.A_UNDERLINE)
stdscr.refresh()
def _update_display(self, stdscr, time_left):
if self.display == "progressive":
self._progressive_display(stdscr, time_left)
elif self.display == "10ff":
self._10ff_display(stdscr, time_left)
def _handle_key(self, key):
char = curses.keyname(key).decode()
if char == "^R":
self.restart()
if key in (curses.KEY_BACKSPACE, 127):
self.input = self.input[:-1]
elif chr(key) == " ":
self._finish_word_event()
else:
self.input += chr(key)
#staticmethod
def _setup_ncurses(stdscr):
# hide cursor
curses.curs_set(0)
# setup colors for printing text to screen
curses.use_default_colors()
curses.init_pair(Game.CORRECT_COLOR, curses.COLOR_GREEN, 0)
curses.init_pair(Game.INCORRECT_COLOR, curses.COLOR_RED, 0)
# don't wait for user input when calling getch()/getkey()
stdscr.nodelay(True)
# allow 100ms sleep on getch()/getkey() avoiding busy-wait
# early returns when key is pressed, meaning no input delay
stdscr.timeout(100)
def _game_loop(self, stdscr):
self._setup_ncurses(stdscr)
self._update_display(stdscr, self.game_time)
started = False
start = time.time()
time_left = self.game_time
while time_left > 0:
if not started:
start = time.time()
key = stdscr.getch()
new_time_left = int(round(self.game_time - (time.time() - start)))
if key == -1:
# only update display when necessary
if time_left != new_time_left:
time_left = new_time_left
self._update_display(stdscr, time_left)
continue
time_left = new_time_left
started = True
self._handle_key(key)
self._update_display(stdscr, time_left)
def print_stats(self):
"""Print ACC/CPM/WPM to console"""
correct = len(self.correct)
total = correct + len(self.incorrect)
accuracy = correct / total * 100
print("ACC: {:.2f}%".format(accuracy))
cpm = self.calculate_cpm(self.game_time)
print("CPM: {:d}".format(cpm))
wpm = self.calculate_wpm(self.game_time)
print("WPM: {:d}".format(wpm))
def restart(self):
"""
Reset the Game class, effective starting a new game
with new words, but based on same configuration.
"""
self.input = ""
self.correct = []
self.incorrect = []
self.typed = []
self.next_words = [self._get_word() for _ in range(QUEUE_SIZE)]
if self.display == "10ff":
self.offset = 0
self.current_line = []
self.next_line = []
self.play()
def play(self):
"""Start typing game and print results to terminal"""
curses.wrapper(self._game_loop)
self.print_stats()
def main():
"""Parse arguments and start game based thereof"""
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
description=textwrap.dedent(
"""\
Start a minimal 10fastfingers-like typing game on the command line.
Keybinds:
CTRL+R: restart
CTRL+C: exit
"""
),
)
## replace this with the select-difficulty + language
parser.add_argument(
"-v",
"--vocab",
type=str,
metavar="vocab-file-path",
default=VOCAB_PATH,
help="path to newline separated vocab file",
)
parser.add_argument(
"-t",
"--game_time",
type=int,
metavar="gametime-seconds",
default=60,
help="duration in seconds of the typing game",
)
parser.add_argument(
"-min",
"--min_length",
type=int,
metavar="min-word-length",
default=1,
help="minimum word length",
)
parser.add_argument(
"-max",
"--max_length",
type=int,
metavar="max-word-length",
default=100,
help="maximum word length",
)
parser.add_argument(
"-w",
"--words",
type=int,
metavar="words-to-read",
default=200,
help="the amount of words to read from vocab - higher increases difficulty",
)
parser.add_argument(
"-a",
"--advanced",
action="store_const",
const=1000,
dest="words",
help="use 1000 most common words (corresponds to 10ff advanced mode)",
)
parser.add_argument(
"-d",
"--display",
type=str,
metavar="display",
default="10ff",
help="how to display words to type '10ff' or 'progressive'",
)
parser.add_argument(
"-m",
"--mode",
type=str,
metavar="mode-selection",
default="word",
help="selecting which mode you wish to play the game in",
)
args = parser.parse_args()
game = Game(args)
try:
game.play()
except KeyboardInterrupt:
pass
If anyone can give me advice on how to fix this that would help. Not sure if its the way I used pyinstaller or if its a problem with how I run a script within the first python file, and dependencies might not be implemented properly. Running the file give me the following:
This was simply a problem with directories. Pyinstaller put the executable file in it's own directory, where it couldn't import from the rest of my python files. I fixed it by dragging the executable into the directory with the rest of my python scripts. This is more of a temporary solution, as the executable has to be in the same directory as the other python files until I fix the import statements.

Co-routine returns None for every alternate iteration

I have a piece of code as shown below:
#!/bin/python3
import math
import os
import random
import re
import sys
import logging
def consumer():
while True:
x = yield
print(x)
def producer(n):
for _ in range(n):
x = int(input())
yield x
def rooter():
logging.info("Running the rooter")
while True:
value = (yield)
yield math.sqrt(value)
def squarer():
logging.info("Running the squarer")
while True:
value = (yield)
print("from squarer: {}".format(value))
yield value * value
def accumulator():
logging.info("Running the accumulator.")
running_total = 0
while True:
value = (yield)
running_total += value
yield running_total
def pipeline(prod, workers, cons):
logging.info("workers: {}".format(workers))
for num in prod:
for i, w in enumerate(workers):
num = w.send(num)
cons.send(num)
for worker in workers:
worker.close()
cons.close()
if __name__ == '__main__':
order = input().strip()
m = int(input())
prod = producer(m)
cons = consumer()
next(cons)
root = rooter()
next(root)
accumulate = accumulator()
next(accumulate)
square = squarer()
next(square)
pipeline(prod, eval(order), cons)
Sample input
[square, accumulate]
3 <- Number of inputs coming further
1 <- actual inputs
2
3
Sample Output
*The output should be as below:*
1
5
14
but comes to
10(sum of the squares of 1 and 3) when it should actually be 14 (sum of the squares of 1, 2, 3)
So essentially the input 2 is missed (It's second in the line of inputs).
On debugging further I found that this is the case for every alternate iteration, not just for the provided inputs here.
I am not able to decipher what's happening. If it's of any help, the co-routine squarer is the one returning None in the second iteration.
I'd appreciate any help.
I found a solution to this.
It's that we prime the co-routine after use in the pipeline function so the code becomes as follows: I have marked the next(w) line within asterix for reference.
#!/bin/python3
import math
import os
import random
import re
import sys
import logging
def consumer():
while True:
x = yield
print(x)
def producer(n):
for _ in range(n):
x = int(input())
yield x
def rooter():
logging.info("Running the rooter")
while True:
value = (yield)
yield math.sqrt(value)
def squarer():
logging.info("Running the squarer")
while True:
value = (yield)
print("from squarer: {}".format(value))
yield value * value
def accumulator():
logging.info("Running the accumulator.")
running_total = 0
while True:
value = (yield)
running_total += value
yield running_total
def pipeline(prod, workers, cons):
logging.info("workers: {}".format(workers))
for num in prod:
for i, w in enumerate(workers):
num = w.send(num)
**next(w)**
cons.send(num)
for worker in workers:
worker.close()
cons.close()
if __name__ == '__main__':
order = input().strip()
m = int(input())
prod = producer(m)
cons = consumer()
next(cons)
root = rooter()
next(root)
accumulate = accumulator()
next(accumulate)
square = squarer()
next(square)
pipeline(prod, eval(order), cons)
As mentioned in PEP specification it says that a generator function's yield
is always None when resumed by a normal next call. So when explicitly made to yield, it'll be ready to handle the next input immediately in this case.

Time issue with ZMQ Recv and recv_string

this code I use below is a ZMQ sub to a publisher that is giving me data. It uses the counter to tell me when its 30 and 59 seconds to run my write to CSV every 30 seconds or so.
Problem: I am now timing all of the processes in my thread. the lines where message and message2 = socket.recv_string() is taking anywhere from half a second to 20 seconds to receive string. Thus causing the thread to miss the 30 and 59 second intervals I set. This was not happening yesterday. The other timers for the if statements are taking .00001 or 0.0 seconds. So that part isnt the problem
Im wondering what could effect this. Could it be the processing power of my computer? Or is the receive string based on how long it waits for the publisher to actually send something?
I'm not running in a dev or production environment and its on a shared server with something like 15 other people and its virtual. A zero client. I've never had this problem before and on another script i have set up for another ZMQ pub/sub I'm receiving messages in .01 or .001 seconds all the way to 3 seconds. Which is more manageable but the norm was .01.
Any tips or help would be amazing. Thanks in advance
import zmq
import pandas as pd
import time
import threading
df_fills = pd.DataFrame()
df_signal = pd.DataFrame()
second_v = [30,59]
s = 0
m = 0
h = 0
d = 0
def counter():
global h,s,m,d
while True:
s += 1
#print("Second:{}".format(s))
if s >=60:
m +=1
s = 0
if m >= 60:
h += 1
m = 0
if h >= 24:
d += 1
h = 0
#print(s)
time.sleep(1)
class zmq_thread(threading.Thread):
def __init__(self,name):
threading.Thread.__init__(self)
self.name = name
def run(self):
global df_fills, second_v,s
print('zmq started')
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect(SERVER)
socket.setsockopt_string(zmq.SUBSCRIBE,'F')
print('socket connected')
tickers = [a bunch of tickers]
while True:
try:
start2 = time.time()
if s == 30:
print('break')
if df_fills.empty == True:
print('running fill thread again')
z = zmq_thread('Start_ZMQ')
#time.sleep(.7)
z.run()
else:
start = time.time()
print('writing fills')
filename = "a CSV"
with open(filename, 'a') as f:
df_fills.to_csv(f, encoding = 'utf-8', index = False, header = False)
f.close()
print('wrote fills')
end = time.time()
print(end-start)
df_fills = df_fills.iloc[0:0]
z = zmq_thread('Start_ZMQ')
z.run()
return df_fills
end2 = time.time()
print(end2-start2)
start3 = time.time()
message = socket.recv_string()
message2 = socket.recv_string()
end3 = time.time()
print(end3-start3, 'message timing')
print(s)
start1 = time.time()
if message == 'F':
# message2_split = message2.split("'")
message2_split = message2.split(";")
message3_split = [e[3:] for e in message2_split]
message4 = pd.Series(message3_split)
if message4[0] in tickers:
df_fills = df_fills.append(message4, ignore_index=True)
print('fill')
end1 = time.time()
print(end1-start1)
except KeyboardInterrupt:
break
counter = threading.Thread(target = counter)
zmq_loop = zmq_thread('Start_ZMQ')
#%%
counter.start()
zmq_loop.start()
I didn't realize that ZMQ typical recv_string is by default blocking. So I did this
message = socket.recv_string(flags = zmq.NOBLOCK)
message2 = socket.recv_string(flags = zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
pass
else:
if message == 'ABA_BB':
message2_split = message2.split(";")
message3_split = [e[3:] for e in message2_split]
message4 = pd.Series(message3_split)
#print(message4)
if message4[2] == '300':
df_signal = df_signal.append(message4, ignore_index=True)
print('Signal Appended')

How do i remove none and implement threading to my desired output

#!/usr/bin/env python
import RPi.GPIO as GPIO
from time import sleep
my objective
"""
this is writen for the raspberry pi to get an input from two encoder and print a numberical
count. this will use with a telescope and stellarium
this need lot more work but just a beginner still
"""
set gpio on raspberry init self and main varible
class EncoderSetup:
def __init__(self, azm_a=27, azm_b=17, lat_a=20, lat_b=21, counter1=0, counter2=0):
self.azm_a = azm_a
self.azm_b = azm_b
self.lat_a = lat_a
self.lat_b = lat_b
self.counter1 = counter1
self.counter2 = counter2
GPIO.setmode(GPIO.BCM)
GPIO.setup(self.azm_a, GPIO.IN)
GPIO.setup(self.azm_b, GPIO.IN)
GPIO.setup(self.lat_a, GPIO.IN)
GPIO.setup(self.lat_b, GPIO.IN)
this loop gets the state of a and b compares, get direction, counts and debounces
class Azimuth(EncoderSetup):
def azm_encoder(self):
Last_RoB_Status = GPIO.input(self.azm_b)
while not GPIO.input(self.azm_a): # starts encoder loop
Current_RoB_Status = GPIO.input(self.azm_b)
dtState = GPIO.input(self.azm_a)
if Current_RoB_Status != Last_RoB_Status:
if dtState != Current_RoB_Status:
self.counter1 += 1
else:
self.counter1 -= 1
Last_RoB_Status = Current_RoB_Status # deboucing
# sleep(0.01)
return self.counter1
this loop gets the state of a and b compares, get direction, counts and debounces
class Latitude(EncoderSetup):
def lat_encoder(self):
Last_RoC_Status = GPIO.input(self.lat_a)
while not GPIO.input(self.lat_b):
Current_RoC_Status = GPIO.input(self.lat_a)
dtState2 = GPIO.input(self.lat_b)
if Current_RoC_Status != Last_RoC_Status:
if dtState2 != Current_RoC_Status:
self.counter2 += 1
else:
self.counter2 -= 1
Last_RoC_Status = Current_RoC_Status # deboucing
# sleep(0.01)
return self.counter2
loop forever to get continous count from two encoders
def loop():
eset = EncoderSetup()
azm = Azimuth()
lat = Latitude()
while True:
print ('globalCounter1 = ' + str(azm.azm_encoder()) +
' globalCounter2 = ' + str(lat.lat_encoder()))
this doc explains my problem
"""
this is my problem i get a numerical output with none example
globalcounter1 = none globalcounter2 none
globalcounter1 = 0 globalcounter2 none
globalcounter1 = none globalcounter2 0
globalcounter1 = none globalcounter2 none
globalcounter1 = 4 globalcounter2 -1
globalcounter1 = 5 globalcounter2 none
the second problem is i got to turn lat_encoder first before
i get and out from azm_encoder. i think i need to implement threading
or the the same loop for both lat_encoder and azm_encoder
please help me i little lost at this point
"""
resets the gpio on the raspberry
def destroy():
GPIO.cleanup()
start the program
if __name__ == '__main__':
use to call loop to start
remove trace back error message
try:
loop()
except KeyboardInterrupt:
destroy()
i solved the first problem. this whats happening
Because there are two print statements. First is inside function and second is outside function. When function not return any thing that time it return None value.
i remove the print from loop() and replaced where return statement are for
example
print self.counter1 and print self.counter2
i wonder if is possible to use the same while loop for both encoders

Thread safe counter using python's multiprocessing starmap

At the moment I'm trying to handle results from a calculation which come in very fast. At first I inserted each simulation result into an sqlite database but it turned out to be the bottleneck of the entire calculation. So I ended up using cursor.executemany instead of cursor.execute which is much faster.
My problem is now that I'm somehow not able to implement a thread safe counter.
The executemany task should be run every 1000 calculations. Therefore I implemented an initializer with a multiprocessing.Value I also tried this solution (http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing) but somehow some values of the counter are duplicates which ends up in running the executemany task to often or not at all.
If anybody has an idea how to solve this issue I'd really appreciate it.
Here's a minimum sample:
import multiprocessing, sqlite3
from multiprocessing import Value, Lock
from itertools import repeat
def worker(Testvalues, TotalValues):
MP_counter.value += 1
counter.increment()
con = sqlite3.connect("Test.db", timeout=30.0)
cur = con.cursor()
# Minimum sample:
Helper = list(range(5))
Helper = [x * Testvalues for x in Helper]
GList.append(Helper)
Execute_Every = 10
print("Counter class: %d" % (counter.value()))
print("MP_counter: %d" % (MP_counter.value))
if counter.value() % Execute_Every == 0 or counter.value() == TotalValues - 1:
print("Execute query")
print("Counter class: %d" % (counter.value()))
print("MP_counter: %d" % (MP_counter.value))
Helper = [tuple(row) for row in GList[:Execute_Every]]
del GList[:Execute_Every]
cur.executemany(
"INSERT INTO Test (One, Two, Three, Four, Five) VALUES (?, ?, ?, ?, ?);", Helper)
con.commit()
con.close()
def setup(t, g, c):
global MP_counter
global GList
global counter
MP_counter = t
GList = g
counter = c
class Counter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
self.lock = Lock()
def increment(self):
with self.lock:
self.val.value += 1
def value(self):
with self.lock:
return self.val.value
if __name__ == '__main__':
m = multiprocessing.Manager()
CPUS = multiprocessing.cpu_count()
MP_counter = multiprocessing.Value('i', 0)
GList = m.list([])
thread_safe_counter = Counter(0)
l = multiprocessing.Lock()
WORKERS = multiprocessing.Pool(initializer=setup, initargs=[MP_counter, GList, thread_safe_counter],processes=CPUS)
con = sqlite3.connect("Test.db", timeout=30.0)
cur = con.cursor()
cur.execute('PRAGMA journal_mode=wal')
SQLCommand = "CREATE TABLE IF NOT EXISTS Test (One INT, Two INT, Three INT, Four INT, Five INT);"
cur.execute(SQLCommand)
con.close()
TotalValues = 100
Testvalues = list(range(TotalValues))
WORKERS.starmap(worker, zip(Testvalues, repeat(TotalValues)))
WORKERS.close()
WORKERS.join()
#Check if list is empty
print(GList)
Thank you guys :)
Your counter has an increment() and a value() method, which need to be called separately, so to make this safe you'd have to call both operations while holding the lock. Your increment() method should return the new value after incrementing it, and you should use that without further calls to value(), e.g:
class Counter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
self.lock = Lock()
def increment(self):
with self.lock:
self.val.value += 1
return self.val.value
...
def worker(Testvalues, TotalValues):
counter_value = counter.increment()
# use only counter_value from here on
...
Also, a Value is already created with a default RLock, which can be overridden in the constructor call with a different lock type if needed. So you don't really need to allocate your own lock, you could just use:
class Counter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
# or Value('i', initval, lock=Lock())
def increment(self):
with self.val.get_lock():
self.val.value += 1
return self.val.value

Resources