Python 将多处理池与psycopg2 ThreadedConnectionPool一起使用

在使用psycopg2.pool中的ThreadedConnectionPool时，我试图从池中获取多个连接，并将它们用于多处理。我创建了以下类来创建一个静态连接池和另一个用于上下文的助手类

import psycopg2
import os
from psycopg2.pool import ThreadedConnectionPool

class Database:
    __connection_pool = None
    
    @staticmethod
    def initialize(**kwargs):
        Database.__connection_pool = ThreadedConnectionPool(
                                                    minconn=1,
                                                    maxconn=15,
                                                    **kwargs)

    @classmethod
    def get_connection(cls):
        try:
            return Database.__connection_pool.getconn()
        except Exception as err:
            print(err)
    
    @classmethod
    def return_connection(cls,connection):
        Database.__connection_pool.putconn(connection,close=False)
        
    @classmethod
    def close_all_connections(cls):
        if not Database.connection_closed():
            Database.__connection_pool.closeall()
        else:
            return "Connection Already Closed!"
    
    @classmethod
    def connection_closed(cls):
        try:
            return Database.__connection_pool.closed
        except:
            return True
        
class CursorFromPool:
    
    '''
    Wrapper class to connect to PostgresDB    
    '''

    def __init__():
        self.key=key
        self.connection = None
        self.cursor = None

    def __enter__(self):
        '''
        using the class with "with"
        call a new connection from the pool
        '''
        self.connection = Database.get_connection()
        self.cursor = self.connection.cursor()
        return self.cursor

    def __exit__(self, exc_type, exc_val, exc_tb):
        '''
        commit the cursor and put the connection
        back to the connection pool
        '''
        if exc_val is not None:
            self.connection.rollback()
        else:
            self.cursor.close()
            self.connection.commit()
        Database.return_connection(self.connection)
        
    def get_open_conn(self):
        self.connection = Database.get_connection()
        if self.connection is not None: 
            self.cursor = self.connection.cursor()
            return self.cursor

    def put_conn(self):
        self.cursor.close()
        self.connection.commit()

    def close_open_connection(self):
        Database.return_connection(self.connection)
        
    def connection_closed(self):
        if self.connection is not None:
            if self.connection.closed == 0:
                return False
            else:
                return True
        else:
            return "No connections established"

上面的类工作得很好，我可以在调用get_open_conn（）时从每个类中检索单个连接，并且我可以从池中耗尽所有连接。在多处理中使用以下代码时，我得到以下错误，即连接已关闭。为了简单起见，我反复调用相同的函数，将相同的数据上传到db中的表中

def clean(skiprows=None):

    df_flags = pd.read_csv('filename.csv')
    data_list = [tuple(row) for row in df_flags.itertuples(index=False)] 
    
    start = time.time()
    with CursorFromPool() as cursor:
        psycopg2.extras.execute_batch(cursor,'INSERT INTO db_name.table_name VALUES (%s,%s,%s,%s)',data)
        print(cursor.statusmessage)
    end = time.time()
    print(start-end)
    return True

Database.initialize(*db_args_passed_here*)
skiper = [1,2,3,4,5,6,7,8]
with Pool(processes = 4) as pool:
    pool.map(clean,skiper)
    
if Database.connection_closed == False:
    Database.close_all_connections()

回溯（最近一次呼叫最后一次）：
文件“/mnt/upload_to_db/helper.py”，第53行，干净
psycopg2.extras.execute\u批处理（游标，'INSERT INTO db\u name.table\u name value（%s，%s，%s）'，数据\u列表）
文件“/usr/local/anaconda/lib/python3.6/site packages/psycopg2/extras.py”，第1198行，在执行批处理中
cur.execute（b“；”.join（sqls））
psycopg2.OperationalError:SSL错误：解密失败或错误记录mac
在处理上述异常期间，发生了另一个异常：
回溯（最近一次呼叫最后一次）：
worker中的文件“/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py”，第119行
结果=（True，func（*args，**kwds））
mapstar中的文件“/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py”，第44行
返回列表（映射（*args））
文件“/mnt/upload_to_db/helper.py”，第54行，干净
打印（游标状态消息）
文件“/mnt/utils/ConnectionPool.py”，第100行，在退出时__
self.connection.rollback（）
psycopg2.InterfaceError:连接已关闭
"""
上述异常是以下异常的直接原因：
回溯（最近一次呼叫最后一次）：
文件“main.py”，第10行，在
游泳池地图（清洁，滑雪者）
文件“/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py”，地图第266行
返回self.\u map\u async（func、iterable、mapstar、chunksize）.get（）
get中的文件“/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py”，第644行
提升自我价值
psycopg2.InterfaceError:连接已关闭

没错，如果您查看数据库类，我定义了一个名为return\u connection的函数。如果您跟踪到CursorFromPool exit方法，它将使用return\u连接退出with上下文。

Traceback (most recent call last):
  File "/mnt/upload_to_db/helper.py", line 53, in clean
    psycopg2.extras.execute_batch(cursor,'INSERT INTO db_name.table_name VALUES (%s,%s,%s,%s)',data_list)
  File "/usr/local/anaconda/lib/python3.6/site-packages/psycopg2/extras.py", line 1198, in execute_batch
    cur.execute(b";".join(sqls))
psycopg2.OperationalError: SSL error: decryption failed or bad record mac


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py", line 119, in worker
    result = (True, func(*args, **kwds))
  File "/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py", line 44, in mapstar
    return list(map(*args))
  File "/mnt/upload_to_db/helper.py", line 54, in clean
    print(cursor.statusmessage)
  File "/mnt/utils/ConnectionPool.py", line 100, in __exit__
    self.connection.rollback()
psycopg2.InterfaceError: connection already closed
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "main.py", line 10, in <module>
    pool.map(clean,skiper)
  File "/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py", line 266, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "/usr/local/anaconda/lib/python3.6/multiprocessing/pool.py", line 644, in get
    raise self._value
psycopg2.InterfaceError: connection already closed