Python 如何在peewee orm中使用ThreadPoolExecutor处理数据库连接池？

我正在脚本中使用和一个mariadb数据库，该脚本查看一个db表，并将该查询中的数据提交到ThreadPoolExecutor。工作者本身也需要查询数据库。在所有的未来都完成之后，脚本将重新开始

当我使用单一连接时，一切正常，但由于我的工作线程主要是网络IO，我觉得所有工作线程的单一连接将成为瓶颈

如果我改为数据库池，我可以监视连接的增加，直到我从peewee得到一个错误“打开的连接太多”。连接本身从不关闭。这适合皮维的

但是，我不知道如何从我的辅助函数内部手动打开和关闭db连接

我尝试将models.py中的数据库变量设置为全局变量，这样我就可以在worker中访问该对象，但观察数据库中所有打开的连接使我意识到.close（）/.open（）在这种情况下没有效果

我还将所有内容粘贴到一个文件中，但仍然无法手动打开/关闭连接

本文仅给出了如何将池与不同的WebFramework一起使用的示例

我的应用程序简化了

#app.py
from models.models import MyTableA, MyTableB

def do_work(data):
    MyTableB.create(name="foo")

def main()
    logger = logging.getLogger()
    data = MyTableA.select().dicts()

    with ThreadPoolExecutor(8) as executor:
        future_to_system = {executor.submit(do_work, d): d.id for d in data}
        
        for future in as_completed(future_to_system):
            system = future_to_system[future]
            try:
                future.result()
            except Exception as exc:
                logger.info('%r generated an exception: %s' % (system, exc))

            else:
                logger.debug('%r result is %s' % (system, "ok"))

if __name__ == '__main__':
    main()

models.py

from peewee import *
from playhouse.pool import PooledMySQLDatabase

#uncomment this line to use pool
#database = PooledMySQLDatabase('db_name', **{'charset': 'utf8', 'sql_mode': 'PIPES_AS_CONCAT', 'use_unicode': True, 'user': 'user', 'passwd': 'pass', 'max_connections': 32, 'stale_timeout': 300})

#comment that line to use pool
database = PooledMySQLDatabase('db_name', **{'charset': 'utf8', 'sql_mode': 'PIPES_AS_CONCAT', 'use_unicode': True, 'user': 'user', 'passwd': 'pass'})




class UnknownField(object):
    def __init__(self, *_, **__): pass

class BaseModel(Model):
    class Meta:
        database = database

class MyTableA(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

class MyTableB(BaseModel):
    name = CharField()


    class Meta:
        table_name = 'my_table'

from peewee import *


class UnknownField(object):
    def __init__(self, *_, **__): pass

class BaseModel(Model):
    class Meta:
        database = None

class MyTableA(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

class MyTableB(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

如果有人知道如何将peewee的connectionpool与Threadpoolexecutor结合使用，我将非常感谢。

我找到了一个解决方案

models.py

from peewee import *
from playhouse.pool import PooledMySQLDatabase

#uncomment this line to use pool
#database = PooledMySQLDatabase('db_name', **{'charset': 'utf8', 'sql_mode': 'PIPES_AS_CONCAT', 'use_unicode': True, 'user': 'user', 'passwd': 'pass', 'max_connections': 32, 'stale_timeout': 300})

#comment that line to use pool
database = PooledMySQLDatabase('db_name', **{'charset': 'utf8', 'sql_mode': 'PIPES_AS_CONCAT', 'use_unicode': True, 'user': 'user', 'passwd': 'pass'})




class UnknownField(object):
    def __init__(self, *_, **__): pass

class BaseModel(Model):
    class Meta:
        database = database

class MyTableA(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

class MyTableB(BaseModel):
    name = CharField()


    class Meta:
        table_name = 'my_table'

from peewee import *


class UnknownField(object):
    def __init__(self, *_, **__): pass

class BaseModel(Model):
    class Meta:
        database = None

class MyTableA(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

class MyTableB(BaseModel):
    name = CharField()

    class Meta:
        table_name = 'my_table'

app.py

from playhouse.pool import PooledMySQLDatabase
from models.models import MyTableA, MyTableB

def do_work(data, db):
    with db:
        MyTableB.create(name="foo")

def main()
    logger = logging.getLogger()
    

    database = PooledMySQLDatabase('db_name', **{'charset': 'utf8', 'sql_mode': 'PIPES_AS_CONCAT', 'use_unicode': True, 'user': 'user', 'passwd': 'pass', 'max_connections': 32, 'stale_timeout': 300})

    database.bind(
        [
            MyTableA, MyTableB
        ]
    )

    with database:
        data = MyTableA.select().dicts()


    with ThreadPoolExecutor(8) as executor:
        future_to_system = {executor.submit(do_work, d): d.id for d in data}
        
        for future in as_completed(future_to_system):
            system = future_to_system[future]
            try:
                future.result()
            except Exception as exc:
                logger.info('%r generated an exception: %s' % (system, exc))

            else:
                logger.debug('%r result is %s' % (system, "ok"))

if __name__ == '__main__':
    main()