Python 3 multiprocessing error - python-3.x

I'm trying to run this code taken form python doc
https://docs.python.org/3.6/library/multiprocessing.html
from multiprocessing import Process
import os
def info(title):
print(title)
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
def f(name):
info('function f')
print('hello', name)
if __name__ == '__main__':
info('main line')
p = Process(target=f, args=('bob',))
p.start()
p.join()
but here what I get:
main line
module name: __main__
parent process: 15744
process id: 7344
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\multiprocessing\spawn.py", line 114, in _main
prepare(preparation_data)
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\multiprocessing\spawn.py", line 225, in prepare
_fixup_main_from_path(data['init_main_from_path'])
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\multiprocessing\spawn.py", line 277, in _fixup_main_from_path
run_name="__mp_main__")
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\runpy.py", line 261, in run_path
code, fname = _get_code_from_file(run_name, path_name)
File "C:\Users\SomeUser\AppData\Local\Programs\Python\Python36-32\lib\runpy.py", line 231, in _get_code_from_file
with open(fname, "rb") as f:
OSError: [Errno 22] Invalid argument: 'C:\\Users\\SomeUser\\<stdin>'
what am I doing wrong? I use python 3.6.1.

Related

Why doesn't this simple multiprocessing function work?

I am just learning about multiprocessing in python. And when I am trying to run the code I receive these problems.
Code
from multiprocessing import Process, cpu_count
def counter1(num):
count = 0
while count < num:
count += 1
def main():
t1_start = perf_counter()
processes = []
for _ in range(4):
p = Process(target=counter1, args=(10000,))
p.start()
processes.append(p)
for process in processes:
process.join()
t1_stop = perf_counter()
print("Elapsed time: {}".format(t1_stop - t1_start))
if __name__ == '__main__':
main()
Problems
Traceback (most recent call last):
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\code.py", line 90, in runcode
exec(code, self.locals)
File "<input>", line 1, in <module>
File "C:\Program Files\JetBrains\PyCharm 2021.3.3\plugins\python\helpers\pydev\_pydev_bundle\pydev_umd.py", line 198, in runfile
pydev_imports.execfile(filename, global_vars, local_vars) # execute the script
File "C:\Program Files\JetBrains\PyCharm 2021.3.3\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "C:/Users/Женя/PycharmProjects/test_loria/mp.py", line 54, in <module>
main()
File "C:/Users/Женя/PycharmProjects/test_loria/mp.py", line 43, in main
p.start()
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <function counter1 at 0x000001FBC61EC040>: attribute lookup counter1 on __main__ failed
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Users\Женя\AppData\Local\Programs\Python\Python310\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
I have no idea what is going on. Because if I try to run the same code in another file, everything will be fine.

pytorch distributed training fails when use 'gloo' backend

I run the distributed training code in the pytorch tutorial, when I use the 'gloo' backend, the code raises RuntimeError when init_process_group.
"""run.py:"""
#!/usr/bin/env python
import os
import torch
import torch.distributed as dist
from torch.multiprocessing import Process
def run(rank, size):
""" Distributed function to be implemented later. """
pass
def init_process(rank, size, fn, backend='gloo'):
""" Initialize the distributed environment. """
os.environ['MASTER_ADDR'] = '127.0.0.1'
os.environ['MASTER_PORT'] = '29500'
dist.init_process_group(backend, rank=rank, world_size=size)
fn(rank, size)
if __name__ == "__main__":
size = 2
processes = []
for rank in range(size):
p = Process(target=init_process, args=(rank, size, run))
p.start()
processes.append(p)
for p in processes:
p.join()
While getting the error:
Process Process-2:
Traceback (most recent call last):
File "/home/wuyiming/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/home/wuyiming/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "<ipython-input-1-6cc8a94a6551>", line 16, in init_process
dist.init_process_group(backend, rank=rank, world_size=size)
File "/home/wuyiming/anaconda3/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 416, in init_process_group
timeout=timeout)
File "/home/wuyiming/anaconda3/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 484, in _new_process_group_helper
timeout=timeout)
RuntimeError: [enforce fail at /pytorch/third_party/gloo/gloo/transport/tcp/device.cc:198] ifa != nullptr. Unable to find interface for: [0.0.0.27]
Process Process-1:
Traceback (most recent call last):
File "/home/wuyiming/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/home/wuyiming/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "<ipython-input-1-6cc8a94a6551>", line 16, in init_process
dist.init_process_group(backend, rank=rank, world_size=size)
File "/home/wuyiming/anaconda3/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 416, in init_process_group
timeout=timeout)
File "/home/wuyiming/anaconda3/lib/python3.7/site-packages/torch/distributed/distributed_c10d.py", line 484, in _new_process_group_helper
timeout=timeout)
RuntimeError: [enforce fail at /pytorch/third_party/gloo/gloo/transport/tcp/device.cc:198] ifa != nullptr. Unable to find interface for: [0.0.0.27]
If I change the backbone from 'gloo' to 'NCCL', the code runs correctly.

Pandas files not found

i can't understand why Pandas not found my fer2013.csv. I have this code:
class DataManager(object):
"""Class for loading fer2013 emotion classification dataset or
imdb gender classification dataset."""
def __init__(self, dataset_name='imdb', dataset_path=None, image_size=(48, 48)):
self.dataset_name = dataset_name
self.dataset_path = dataset_path
self.image_size = image_size
if self.dataset_path != None:
self.dataset_path = dataset_path
elif self.dataset_name == 'imdb':
self.dataset_path = '../datasets/imdb_crop/imdb.mat'
elif self.dataset_name == 'fer2013':
self.dataset_path = '../datasets/fer2013/fer2013.csv'
elif self.dataset_name == 'KDEF':
self.dataset_path = '../datasets/KDEF/'
else:
raise Exception('Incorrect dataset name, please input imdb or fer2013')
I get this error:
Traceback (most recent call last):
File "train_model.py", line 55, in <module>
faces, emotions = data_loader.get_data()
File "C:\Users\devel\Documents\Proyectos\erecog\utils\datasets.py", line 31, in get_data
ground_truth_data = self._load_fer2013()
File "C:\Users\devel\Documents\Proyectos\erecog\utils\datasets.py", line 57, in _load_fer2013
data = pd.read_csv(self.dataset_path)
File "C:\Users\devel\Documents\Proyectos\erecog\env\lib\site-packages\pandas\io\parsers.py", line 676, in parser_f
return _read(filepath_or_buffer, kwds)
File "C:\Users\devel\Documents\Proyectos\erecog\env\lib\site-packages\pandas\io\parsers.py", line 448, in _read
parser = TextFileReader(fp_or_buf, **kwds)
File "C:\Users\devel\Documents\Proyectos\erecog\env\lib\site-packages\pandas\io\parsers.py", line 880, in __init__
self._make_engine(self.engine)
File "C:\Users\devel\Documents\Proyectos\erecog\env\lib\site-packages\pandas\io\parsers.py", line 1114, in _make_engine
self._engine = CParserWrapper(self.f, **self.options)
File "C:\Users\devel\Documents\Proyectos\erecog\env\lib\site-packages\pandas\io\parsers.py", line 1891, in __init__
self._reader = parsers.TextReader(src, **kwds)
File "pandas\_libs\parsers.pyx", line 374, in pandas._libs.parsers.TextReader.__cinit__
File "pandas\_libs\parsers.pyx", line 674, in pandas._libs.parsers.TextReader._setup_parser_source
FileNotFoundError: [Errno 2] File ../datasets/fer2013/fer2013.csv does not exist: '../datasets/fer2013/fer2013.csv'
I have that file in that folder, i don't know why it not found my file.
I solved this error. Thanks to #DYZ i could see the directory where the process was working, the process was out the 'utils' folder. I changed the route to './datasets/fer2013/fer2013.csv' with the point '.' before the '/'.

Python3 multiprocessing shared dictionary consume by all process

I'm beginner for multiprocessing,
i would like to use multiprocessing for parallel code running with streaming data.
To start good, I have coded below and got error.
Could you please tell me correct way to print on the screen.
Code:
import sys
from multiprocessing import Process, Manager
import time
def producer(dic, name):
for i in range(10000):
dic["A"] = i
time.sleep(2)
def consumer(dic, name):
for i in range(10000):
aval = dic.get("A")
#print(f" {name} - Val = {aval}")
sys.stdout.write(f" {name} - Val = {aval}")
sys.stdout.flush()
time.sleep(2.2)
if __name__ == '__main__':
manager = Manager()
dic = manager.dict()
Process(target=producer, args=(dic,"TT")).start()
time.sleep(1)
Process(target=consumer, args=(dic,"Con1")).start()
Process(target=consumer, args=(dic,"Con2")).start()
When I run the same in the windows console, I got below error, how can I print Consumer's print function in the console.Thanks
(base) PS D:\> python .\mulpro.py
Process Process-3:
Process Process-4:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 811, in
_callmethod
conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 19, in consumer
aval = dic.get("A")
File "<string>", line 2, in get
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 815, in
_callmethod
self._connect()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 802, in
_connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 490, i
n Client
c = PipeClient(address)
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 691, i
n PipeClient
_winapi.WaitNamedPipe(address, 1000)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 811, in
_callmethod
conn = self._tls.connection
FileNotFoundError: [WinError 2] The system cannot find the file specified
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 19, in consumer
aval = dic.get("A")
File "<string>", line 2, in get
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 815, in
_callmethod
self._connect()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 802, in
_connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 490, i
n Client
c = PipeClient(address)
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 691, i
n PipeClient
_winapi.WaitNamedPipe(address, 1000)
FileNotFoundError: [WinError 2] The system cannot find the file specified
Process Process-2:
Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 297, in _
bootstrap
self.run()
File "C:\ProgramData\Anaconda3\lib\multiprocessing\process.py", line 99, in ru
n
self._target(*self._args, **self._kwargs)
File "D:\mulpro.py", line 13, in producer
dic["A"] = i
File "<string>", line 2, in __setitem__
File "C:\ProgramData\Anaconda3\lib\multiprocessing\managers.py", line 818, in
_callmethod
conn.send((self._id, methodname, args, kwds))
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 206, i
n send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\ProgramData\Anaconda3\lib\multiprocessing\connection.py", line 280, i
n _send_bytes
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
BrokenPipeError: [WinError 232] The pipe is being closed
The reason might be that the main process doesn't wait for two children process, so make your code like this could work:
def run():
manager = Manager()
dic = manager.dict()
Process(target=producer, args=(dic,"TT")).start()
time.sleep(1)
Process(target=consumer, args=(dic,"Con1")).start()
Process(target=consumer, args=(dic,"Con2")).start()
while True:
pass
if __name__ == '__main__':
run()
However it's very strange that if I append a dead loop in main instead of using another function, it still raise that exception. Anyway, the code above could help.

Python Module attribute error

I know this topic has been came up many times, but I am totally stuck and need help. Please tell me what I have done wrong here and how to fix it. Thanks in advance.
# testcase1.py
import unittest
import sys
class Global:
b = 0
class Util_Case_ID(unittest.TestCase):
def setUp(self):
sys.path.insert(0, 'C:/**/views')
sys.path.insert(0, 'C:/**/app')
sys.path.insert(0, 'C:/**/tests')
from app.views.utility import method_a
Global.b = len(method_a())
def test1(self):
self.assertEqual(Global.b, 11)
def tearDown(self):
sys.path.remove('C:/***/app/views')
sys.path.remove('C:/***/app')
sys.path.remove('C:/*/tests')
if __name__ == "__main__":
unittest.main()
AttributeError: module 'UnitTests' has no attribute 'testcase1'
The Traceback is as follows:
Traceback (most recent call last):
File "C:\Users\*****\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\utrunner.py", line 167, in <module>
all.addTests(testLoader.loadTestsFromTestClass(getattr(module, a[1])),
File "C:\Users\*****\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\nose_helper\loader.py", line 108, in loadTestsFromTestClass
return self.suiteClass(ContextList(cases, context=cls))
File "C:\Users\*****\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\nose_helper\suite.py", line 253, in __call__
return self.makeSuite(tests, context, **kw)
File "C:\Users\*****\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\nose_helper\suite.py", line 291, in makeSuite
for ancestor in self.ancestry(context):
File "C:\Users\******\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\nose_helper\suite.py", line 269, in ancestry
yield resolve_name('.'.join(ancestors))
File "C:\Users\*****\AppData\Roaming\JetBrains\PyCharm Edu 3.5.1\helpers\pycharm\nose_helper\util.py", line 70, in resolve_name
obj = getattr(obj, part)
AttributeError: module 'UnitTests' has no attribute 'testcase1'
Process finished with exit code 1

Resources