Python多处理-如何实现共享计数器和队列

我有以下Python3代码：

from multiprocessing import Process, JoinableQueue, Value
from multiprocessing.managers import SyncManager

def test():
    global numfail
    global queue
    while not queue.empty():
        number = queue.get()
        with numfail.get_lock():
            numfail.value += number
        queue.task_done()

def startthreads():
    global numfail
    global queue
    queue = JoinableQueue()
    for i in range(100):
        numfail = Value('i',0)
        for j in range(10):
            queue.put(j)
        for j in range(10):
            Process(target=test, args=[]).start()

        queue.join()
        print(i, numfail.value)

numfail = 0
startthreads()

每次的输出应该是45，但是有时一些值不能加到和中，有时会出现死锁

我还与一位经理一起尝试过：

from multiprocessing import Process, JoinableQueue, Value
from multiprocessing.managers import SyncManager

def test():
    global numfail
    global queue
    while not queue.empty():
        number = queue.get()
        numfail.value += number
        queue.task_done()

def startthreads():
    global numfail
    global queue
    queue = JoinableQueue()
    manager = SyncManager()
    manager.start()
    for i in range(100):
        numfail = manager.Value('i',0)
        for j in range(10):
            queue.put(j)
        for j in range(10):
            Process(target=test, args=[]).start()

        queue.join()
        print(i, numfail.value)

numfail = 0
startthreads()

但是我得到了下面的错误

Process Process-281:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "MT.py", line 9, in test
    numfail.value += number
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 1154, in get
    return self._callmethod('get')
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 850, in _callmethod
    raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError: 
---------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 250, in serve_client
    self.id_to_local_proxy_obj[ident]
KeyError: '7f8da01ff310'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 252, in serve_client
    raise ke
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 246, in serve_client
    obj, exposed, gettypeid = id_to_obj[ident]
KeyError: '7f8da01ff310'
---------------------------------------------------------------------------
Process Process-280:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "MT.py", line 9, in test
    numfail.value += number
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 1154, in get
    return self._callmethod('get')
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 850, in _callmethod
    raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError: 
---------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 250, in serve_client
    self.id_to_local_proxy_obj[ident]
KeyError: '7f8da01ff310'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 252, in serve_client
    raise ke
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 246, in serve_client
    obj, exposed, gettypeid = id_to_obj[ident]
KeyError: '7f8da01ff310'
---------------------------------------------------------------------------
Process Process-275:
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
    self.run()
  File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "MT.py", line 9, in test
    numfail.value += number
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 1154, in get
    return self._callmethod('get')
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 850, in _callmethod
    raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError: 
---------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 250, in serve_client
    self.id_to_local_proxy_obj[ident]
KeyError: '7f8da01ff310'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 252, in serve_client
    raise ke
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 246, in serve_client
    obj, exposed, gettypeid = id_to_obj[ident]
KeyError: '7f8da01ff310'
---------------------------------------------------------------------------

---

如何解决问题，或者如何以另一种方式实现，以便将队列中的“任务”发送给另一个函数，让该函数处理所有项，并为每个项增加一个变量。

去掉全局变量，将它们作为参数传递给子进程

此外，您还可以将SyncManager代理用于队列：

queue = manager.Queue()

也可以考虑使用游泳池