I am playing around with the Process library from multiprocessing and I was trying to call a class from another file as a separate process, however, I am getting this error:
Traceback (most recent call last):
File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
TypeError: 'tuple' object is not callable
and it still exists in the same process as my stdout is this:
20472 __main__
20472 CALL
internal func call
Leaving Call
Process Process-1:
#This is where the error prints out
Leaving main
where 20472 is the pid.
Main File:
import CALL
from multiprocessing import Process
import os
if __name__ == '__main__':
print(os.getpid(),__name__)
p = Process(target=(CALL.Call(),))
p.start()
p.join()
print("Leaving main")
import os
Call Class File:
class Call():
def __init__(self):
print(os.getpid(), __name__)
self.internal()
def __exit__(self):
print("Leaving Call")
def internal(self):
print("internal func call")
self.__exit__()
As answered in the comments by #Jeronimo - changing
p = Process(target=(call.Call(),)
to
p = Process(target=(call.Call))
was the solution giving the correct output:
2965 __main__
2966 CALL
internal func call
Leaving Call
Leaving main
with a separate process for the called class.
Related
I'm trying to get a PyQt lineEdit widget to place its' contents within a variable in the module 'followup' but for some reason the module does not recognize the PyQt MainWindow as being properly instantiated.
#workbot_main.py
import gspread
import os
import sys
sys.path.append(os.path.abspath(r"C:\Users\leamy\Desktop\work bot\workbot.py"))
sys.path.append(os.path.abspath(r"C:\Users\leamy\Desktop\work bot\initial.py"))
sys.path.append(os.path.abspath(r"C:\Users\leamy\Desktop\work bot\followup.py"))
import threading
import followup
import initial
from PyQt5 import QtWidgets
from PyQt5 import QtCore
from PyQt5.QtWidgets import QMainWindow, QApplication
from workbot import *
import workbot
#workbot.py is the QTdesigner file containing the widgets,
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
followup_start_button = self.ui.followup_start_button
followup_start_button.clicked.connect(threads.begin_initialize)
def followup_starting_number(self):
num = self.ui.lineEdit.text()
return num
def begin_initialize(self):
t1 = threading.Thread(target = followup.initialize, args = ())
t1.start()
threads = Threads()
if __name__ == '__main__':
app = QtWidgets.QApplication([])
widget = MainWindow()
widget.show()
app.exec_()
#followup.py
import workbot_main
def initialize():
row_number.number = workbot_main.MainWindow.followup_starting_number()
print(row_number.number)
I get this error
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Users\leamy\anaconda3\lib\threading.py", line 973, in _bootstrap_inner
self.run()
File "C:\Users\leamy\anaconda3\lib\threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\leamy\Desktop\work bot\followup.py", line 313, in initialize
row_number.number = workbot_main.MainWindow.followup_starting_number()
TypeError: followup_starting_number() missing 1 required positional argument: 'self'
I can't understand why it doesn't work.
Edit:
If I try to use
row_number.number = workbot_main.widget.followup_starting_number()
instead of
row_number.number = workbot_main.MainWindow.followup_starting_number()
I get this error
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Users\leamy\anaconda3\lib\threading.py", line 973, in _bootstrap_inner
self.run()
File "C:\Users\leamy\anaconda3\lib\threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\leamy\Desktop\work bot\followup.py", line 314, in initialize
row_number.number = workbot_main.widget.followup_starting_number()
AttributeError: module 'workbot_main' has no attribute 'widget'
I don't know if that is related.
from asyncio import FIRST_EXCEPTION
from concurrent.futures.thread import ThreadPoolExecutor
from queue import Queue
from concurrent.futures import wait
import os
def worker(i: int, in_queue: Queue) -> None:
while 1:
data = in_queue.get()
if data is None:
in_queue.put(data)
print(f'worker {i} exit')
return
print(os.path.exists(data))
def main():
with ThreadPoolExecutor(max_workers=2) as executor:
queue = Queue(maxsize=2)
workers = [executor.submit(worker, i, queue) for i in range(2)]
for obj in [{'fn': '/path/to/sth'}, {}]:
fn = obj['fn'] # here is the exception
queue.put((fn,))
queue.put(None)
done, error = wait(workers, return_when=FIRST_EXCEPTION)
print(done, error)
main()
This program get stuck when exception happens.
From the log:
Traceback (most recent call last):
File "test.py", line 34, in <module>
main()
File "test.py", line 31, in main
print(done, error)
File "/.pyenv/versions/3.7.4/lib/python3.7/concurrent/futures/_base.py", line 623, in __exit__
self.shutdown(wait=True)
File "/.pyenv/versions/3.7.4/lib/python3.7/concurrent/futures/thread.py", line 216, in shutdown
t.join()
File "/.pyenv/versions/3.7.4/lib/python3.7/threading.py", line 1044, in join
self._wait_for_tstate_lock()
File "/.pyenv/versions/3.7.4/lib/python3.7/threading.py", line 1060, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt
It happens because wait function keep locked, but it's weird because the exception happens before the wait function. It should be returned when exception happens!!
Why it doesn't return immediately when exception happens?
Yes, I find the reason. When the exception happens, executor will exit, and in with statement, it calls self.shutdown(wait=True), so main thread waits for sub thread to exit, however sub thread keep running.
So the solution is that, shutdown the executor manually with wait=False：
try:
## code here
except Exception as e:
traceback.print_exc()
executor.shutdown(False)
I am learning multithreading in python. can anyone pleae tell me why the thread does not start?
code:
import threading
import time
import logging
class Threads_2:
def __new__(cls):
"""
this will be invoked once the creation procedure of the object begins
"""
instance = super(Threads_2,cls).__new__(cls)
return instance
def __init__(self):
"""
this will be invoked once the initialisation procedure of the object begins
"""
#self.configLogging()
#threadinf.Thread.__init__(self)
#self.spawnThreads()
def spawnThreads(self):
if __name__ == "__main__":
thread1 = threading.Thread(target=self.backgroundTask, args=(10,))
thread1.start()
def backgroundTask(numOfLoops):
for i in numOfLoops:
print(2)
obj = Threads_2()
obj.spawnThreads()
error:
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Users\xxx\AppData\Local\Programs\Python\Python39\lib\threading.py", line 954, in _bootstrap_inner
self.run()
File "C:\Users\xxx\AppData\Local\Programs\Python\Python39\lib\threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
TypeError: backgroundTask() takes 1 positional argument but 2 were given
PS D:\python workspace>
The function backgroundTask is in the class Threads_2. So, it should be backgroundTask(self, numOfLoops) instead of backgroundTask(numOfLoops).
This snippet of code (a minimal server running in a thread, code taken from here) works fine with Python3.8.3 but raises an error message with Python3.9.0:
import asyncio
import threading
from aiohttp import web
def aiohttp_server():
def say_hello(request):
return web.Response(text='Hello, world')
app = web.Application()
app.add_routes([web.get('/', say_hello)])
runner = web.AppRunner(app)
return runner
def run_server(runner):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(runner.setup())
site = web.TCPSite(runner, 'localhost', 8080)
loop.run_until_complete(site.start())
loop.run_forever()
t = threading.Thread(target=run_server, args=(aiohttp_server(),))
t.start()
The error message:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.9/threading.py", line 954, in _bootstrap_inner
self.run()
File "/usr/lib/python3.9/threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
File "/home/alkhinoos/nikw/nikw/z2.py", line 21, in run_server
loop.run_until_complete(site.start())
File "/usr/lib/python3.9/asyncio/base_events.py", line 642, in run_until_complete
return future.result()
File "/usr/lib/python3.9/site-packages/aiohttp/web_runner.py", line 121, in start
self._server = await loop.create_server(
File "/usr/lib/python3.9/asyncio/base_events.py", line 1460, in create_server
infos = await tasks.gather(*fs, loop=self)
File "/usr/lib/python3.9/asyncio/base_events.py", line 1400, in _create_server_getaddrinfo
infos = await self._ensure_resolved((host, port), family=family,
File "/usr/lib/python3.9/asyncio/base_events.py", line 1396, in _ensure_resolved
return await loop.getaddrinfo(host, port, family=family, type=type,
File "/usr/lib/python3.9/asyncio/base_events.py", line 856, in getaddrinfo
return await self.run_in_executor(
File "/usr/lib/python3.9/asyncio/base_events.py", line 809, in run_in_executor
executor = concurrent.futures.ThreadPoolExecutor(
File "/usr/lib/python3.9/concurrent/futures/__init__.py", line 49, in __getattr__
from .thread import ThreadPoolExecutor as te
File "/usr/lib/python3.9/concurrent/futures/thread.py", line 37, in <module>
threading._register_atexit(_python_exit)
File "/usr/lib/python3.9/threading.py", line 1374, in _register_atexit
raise RuntimeError("can't register atexit after shutdown")
RuntimeError: can't register atexit after shutdown
What's going on ? The same problem appears with Python 3.9.1. Is this problem solved with Python 3.9.2 ? Maybe a relative issue here.
As mentioned in Python manual - Thread Objects
Other threads can call a thread’s join() method. This blocks the calling thread until the thread whose join() method is called is terminated.
After calling t.start() in main thread, the main thread will exit. Then the process is ended.
If you want to run the child thread forever or until it exits, you should call t.join() in main thread after t.start().
Not sure what you did, but I used 127.0.0.1 instead of localhost and the error is resolved!
I managed to get this sorted by importing the ThreadPoolExecutor module, before calling any of my main application code - so at the top of my main.py start-up script.
thread_pool_ref = concurrent.futures.ThreadPoolExecutor
Just the act of importing the module early (before any threads are initialised) was enough to fix the error. There is an on-going issue around how this module must be present in the main thread, before any child threads import or use any related threading library code.
The inspiration for this fix came from this post on the Python bugs site. My issue was specifically around boto3 library, but the fix is applicable across the board.
I am trying to have a simple subclass of OrderedDict that gets created by a Pool then returned.
It seems that the pickling process when returning the created object to the pool tries to re-instantiate the object and fails due to the required additional argument in the __init__ function.
This is a minimal (non) working example:
from collections import OrderedDict
from multiprocessing import Pool
class Obj1(OrderedDict):
def __init__(self, x, *args, **kwargs):
super().__init__(*args, **kwargs)
self.x = x
def task(x):
obj1 = Obj1(x)
return obj1
if __name__ == '__main__':
with Pool(1) as pool:
for x in pool.imap_unordered(task, (1,2,3)):
print(x.x)
If I do this I get the following error.
Exception in thread Thread-3:
Traceback (most recent call last):
File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.6/multiprocessing/pool.py", line 463, in _handle_results
task = get()
File "/usr/lib/python3.6/multiprocessing/connection.py", line 251, in recv
return _ForkingPickler.loads(buf.getbuffer())
TypeError: init() missing 1 required positional argument: 'x'
Again this fails when the task functions returns to the pool and I guess the object gets pickled?
If I changed OrderedDict by a simple dict it works flawlessly....
I have a workaround to use kwargs and retrieve the attribute of interest but I am stumped about the error to start with. Any ideas?
You can define __getstate__() and __setstate__() methods for your class.
In those functions you can make sure that x is handled as well. For example:
def __getstate__(self):
return self.x, self.items()
def __setstate__(self, state):
self.x = state[0]
self.update(state[1])
BTW, from CPython 3.6 there is no reason to use OrderedDict, since dictionary order is insertion order. This was originally an implementation detail in CPython. In Python 3.7 it was made part of the language.