Python SQLAlchemy-尝试将大BLOB（即机器学习模型）存储到MySQL数据库时发生管道断裂错误

我想在MySQL数据库中序列化许多Gensim库Word2Vec模型。在我的项目中，我正在使用：

Python

、

Flask

、

SQLAlchemy

、和

MySQL

+

PyMySQL

。下面您可以看到我的

Word2VecModel

类，我使用它在MySQL数据库中创建word2vec表：

from db.db import db
import db_models.influencer

class Word2VecModel(db.Model):
    __tablename__ = "word2vec"

    m_id = db.Column(db.Integer, primary_key=True)
    m_username = db.Column(db.String(20), db.ForeignKey("influencer.m_username"))
    m_binary = db.Column(db.LargeBinary)
    influencer = db.relationship("InfluencerModel")

    def __init__(self, m_binary, m_username):
        self.m_binary = m_binary
        self.m_username = m_username

    def __repr__(self):
        return '<Word2Vec {}>'.format(self.m_name)

    @classmethod
    def find_by_username(cls, m_username):
        return cls.query.filter_by(name=m_username).first()

    def save_to_db(self):
        db.session.add(self)
        db.session.commit()

    def delete_from_db(self):
        db.session.delete(self)
        db.session.commit()

我收到以下错误消息：

Traceback (most recent call last):
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 705, in _write_bytes
    self._sock.sendall(data)
BrokenPipeError: [Errno 32] Broken pipe

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
    cursor, statement, parameters, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
    cursor.execute(statement, parameters)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
    result = self._query(query)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
    self._execute_command(COMMAND.COM_QUERY, sql)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
    self.write_packet(sql[:packet_size])
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
    self._write_bytes(data)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
    "MySQL server has gone away (%r)" % (e,))
pymysql.err.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))")

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "app.py", line 117, in <module>
    word2vec.save_to_db()
  File "/home/stefan/PycharmProjects/NLP - Influencer Text Analysis/src/flask_api/db_models/word2vec.py", line 25, in save_to_db
    db.session.commit()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 163, in do
    return getattr(self.registry(), name)(*args, **kwargs)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1042, in commit
    self.transaction.commit()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 504, in commit
    self._prepare_impl()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl
    self.session.flush()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2523, in flush
    self._flush(objects)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush
    transaction.rollback(_capture_exception=True)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
    exc_value, with_traceback=exc_tb,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush
    flush_context.execute()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
    rec.execute(self)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
    uow,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
    insert,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1136, in _emit_insert_statements
    statement, params
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
    return meth(self, multiparams, params)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1133, in _execute_clauseelement
    distilled_params,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
    e, statement, parameters, cursor, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
    sqlalchemy_exception, with_traceback=exc_info[2], from_=e
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
    cursor, statement, parameters, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
    cursor.execute(statement, parameters)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
    result = self._query(query)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
    self._execute_command(COMMAND.COM_QUERY, sql)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
    self.write_packet(sql[:packet_size])
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
    self._write_bytes(data)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
    "MySQL server has gone away (%r)" % (e,))
Killed

---

Killed

->您确定您的进程没有耗尽内存吗？我真的不确定。我如何追踪它？顺便说一下，我看到我的文件大小是770MB，所以我尝试使用joblib以某种方式压缩它。将ML模型存储到DB时，最佳做法是什么？

Killed

->您确定您的进程没有耗尽内存吗？我真的不确定。我如何追踪它？顺便说一下，我看到我的文件大小是770MB，所以我尝试使用joblib以某种方式压缩它。在将ML模型存储到DB时，最佳实践是什么？

Traceback (most recent call last):
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 705, in _write_bytes
    self._sock.sendall(data)
BrokenPipeError: [Errno 32] Broken pipe

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
    cursor, statement, parameters, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
    cursor.execute(statement, parameters)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
    result = self._query(query)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
    self._execute_command(COMMAND.COM_QUERY, sql)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
    self.write_packet(sql[:packet_size])
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
    self._write_bytes(data)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
    "MySQL server has gone away (%r)" % (e,))
pymysql.err.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))")

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "app.py", line 117, in <module>
    word2vec.save_to_db()
  File "/home/stefan/PycharmProjects/NLP - Influencer Text Analysis/src/flask_api/db_models/word2vec.py", line 25, in save_to_db
    db.session.commit()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 163, in do
    return getattr(self.registry(), name)(*args, **kwargs)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 1042, in commit
    self.transaction.commit()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 504, in commit
    self._prepare_impl()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 483, in _prepare_impl
    self.session.flush()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2523, in flush
    self._flush(objects)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2664, in _flush
    transaction.rollback(_capture_exception=True)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 69, in __exit__
    exc_value, with_traceback=exc_tb,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2624, in _flush
    flush_context.execute()
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
    rec.execute(self)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
    uow,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
    insert,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 1136, in _emit_insert_statements
    statement, params
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1014, in execute
    return meth(self, multiparams, params)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1133, in _execute_clauseelement
    distilled_params,
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
    e, statement, parameters, cursor, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
    sqlalchemy_exception, with_traceback=exc_info[2], from_=e
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
    raise exception
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
    cursor, statement, parameters, context
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
    cursor.execute(statement, parameters)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 163, in execute
    result = self._query(query)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/cursors.py", line 321, in _query
    conn.query(q)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 504, in query
    self._execute_command(COMMAND.COM_QUERY, sql)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 773, in _execute_command
    self.write_packet(sql[:packet_size])
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 634, in write_packet
    self._write_bytes(data)
  File "/home/stefan/.virtualenvs/dl4cv/lib/python3.6/site-packages/pymysql/connections.py", line 710, in _write_bytes
    "MySQL server has gone away (%r)" % (e,))
Killed

app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://root:''@localhost/********'
app.config['PROPAGATE_EXCEPTIONS'] = True
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_recycle": 120}