Python 芹菜-Heroky上的Django web应用程序,带有芹菜工人进程-我如何追踪内存泄漏?
我正在运行heroku日志--ps芹菜--tail并看到以下内容:Python 芹菜-Heroky上的Django web应用程序,带有芹菜工人进程-我如何追踪内存泄漏?,python,heroku,celery,django-celery,Python,Heroku,Celery,Django Celery,我正在运行heroku日志--ps芹菜--tail并看到以下内容: Error R14 (Memory quota exceeded) Process running mem=650M(127.1%) DEBUG设置为False 芹菜的版本是3.1.9 我不知道如何追查这个内存泄漏。我采取的第一步是打印出所有与芹菜相关的配置设置,粘贴在下面。我对如何调试这个问题的其他想法很感兴趣。我可以检查芹菜任务本身使用了多少内存吗?我必须明确地告诉它释放内存吗?还有其他想法吗 芹菜配置设置: {
Error R14 (Memory quota exceeded)
Process running mem=650M(127.1%)
DEBUG
设置为False
芹菜的版本是3.1.9
我不知道如何追查这个内存泄漏。我采取的第一步是打印出所有与芹菜相关的配置设置,粘贴在下面。我对如何调试这个问题的其他想法很感兴趣。我可以检查芹菜任务本身使用了多少内存吗?我必须明确地告诉它释放内存吗?还有其他想法吗
芹菜配置设置:
{
"CELERY_RESULT_DB_TABLENAMES": null,
"BROKER_TRANSPORT_OPTIONS": {},
"BROKER_USE_SSL": false,
"CELERY_BROADCAST_QUEUE": "celeryctl",
"CELERY_STORE_ERRORS_EVEN_IF_IGNORED": false,
"CELERY_CREATE_MISSING_QUEUES": true,
"CELERY_DEFAULT_QUEUE": "celery",
"CELERY_SEND_TASK_SENT_EVENT": false,
"CELERYD_TASK_TIME_LIMIT": null,
"CELERY_EVENT_QUEUE_EXPIRES": null,
"CELERY_DEFAULT_EXCHANGE_TYPE": "direct",
"CELERYBEAT_SCHEDULER": "djcelery.schedulers.DatabaseScheduler",
"CELERY_MAX_CACHED_RESULTS": 5000,
"CELERY_RESULT_PERSISTENT": null,
"CELERYD_POOL": "prefork",
"CELERYD_AGENT": null,
"CELERY_CACHE_BACKEND_OPTIONS": {},
"BROKER_HEARTBEAT": null,
"CELERY_RESULT_ENGINE_OPTIONS": null,
"CELERY_RESULT_SERIALIZER": "json",
"CELERYBEAT_SCHEDULE_FILENAME": "celerybeat-schedule",
"CELERY_REDIRECT_STDOUTS_LEVEL": "WARNING",
"CELERY_IMPORTS": [],
"CELERYD_TASK_LOG_FORMAT": "[%(asctime)s: %(levelname)s/%(processName)s] %(task_name)s[%(task_id)s]: %(message)s",
"CELERY_SECURITY_CERTIFICATE": null,
"CELERYD_LOG_COLOR": null,
"CELERY_RESULT_EXCHANGE": "celeryresults",
"CELERY_TRACK_STARTED": false,
"CELERY_REDIS_PASSWORD": null,
"BROKER_USER": null,
"CELERY_COUCHBASE_BACKEND_SETTINGS": null,
"CELERY_RESULT_EXCHANGE_TYPE": "direct",
"CELERY_REDIS_DB": null,
"CELERYD_TIMER_PRECISION": 1.0,
"CELERY_REDIS_PORT": null,
"BROKER_TRANSPORT": null,
"CELERYMON_LOG_FILE": null,
"CELERYD_CONCURRENCY": 0,
"CELERYD_HIJACK_ROOT_LOGGER": true,
"BROKER_VHOST": null,
"CELERY_DEFAULT_EXCHANGE": "celery",
"CELERY_DEFAULT_ROUTING_KEY": "celery",
"CELERY_ALWAYS_EAGER": false,
"CELERYD_TASK_SOFT_TIME_LIMIT": null,
"CELERY_WORKER_DIRECT": false,
"CELERY_REDIS_HOST": null,
"CELERY_QUEUE_HA_POLICY": null,
"BROKER_PORT": null,
"CELERYD_AUTORELOADER": "celery.worker.autoreload:Autoreloader",
"BROKER_CONNECTION_TIMEOUT": 4,
"CELERY_ENABLE_REMOTE_CONTROL": true,
"CELERY_RESULT_DB_SHORT_LIVED_SESSIONS": false,
"CELERY_EVENT_SERIALIZER": "json",
"CELERY_REDIS_MAX_CONNECTIONS": null,
"CELERY_CACHE_BACKEND": null,
"CELERYD_PREFETCH_MULTIPLIER": 4,
"BROKER_PASSWORD": null,
"CELERY_MONGODB_BACKEND_SETTINGS": null,
"CELERY_EAGER_PROPAGATES_EXCEPTIONS": false,
"CELERY_IGNORE_RESULT": false,
"CELERYMON_LOG_LEVEL": "INFO",
"CELERY_DISABLE_RATE_LIMITS": false,
"CELERY_TASK_PUBLISH_RETRY_POLICY": {
"interval_start": 0,
"interval_max": 1,
"max_retries": 3,
"interval_step": 0.2
},
"CELERY_SECURITY_KEY": null,
"CELERY_BROADCAST_EXCHANGE_TYPE": "fanout",
"CELERY_DEFAULT_RATE_LIMIT": null,
"CELERYD_POOL_PUTLOCKS": true,
"CELERY_TASK_SERIALIZER": "json",
"CELERYD_WORKER_LOST_WAIT": 10.0,
"CELERYD_POOL_RESTARTS": false,
"CELERY_TASK_PUBLISH_RETRY": true,
"CELERY_ENABLE_UTC": true,
"CELERY_SEND_EVENTS": false,
"BROKER_CONNECTION_MAX_RETRIES": 100,
"CELERYD_LOG_FILE": null,
"CELERYD_FORCE_EXECV": false,
"CELERY_CHORD_PROPAGATES": true,
"CELERYD_AUTOSCALER": "celery.worker.autoscale:Autoscaler",
"CELERYD_STATE_DB": null,
"CELERY_ROUTES": null,
"BROKER_HEARTBEAT_CHECKRATE": 3.0,
"CELERY_ACCEPT_CONTENT": [
"json"
],
"BROKER_LOGIN_METHOD": null,
"BROKER_CONNECTION_RETRY": true,
"CELERY_TIMEZONE": null,
"CELERYBEAT_MAX_LOOP_INTERVAL": 0,
"CELERYD_LOG_LEVEL": "WARN",
"CELERY_REDIRECT_STDOUTS": true,
"BROKER_POOL_LIMIT": 10,
"CELERY_SECURITY_CERT_STORE": null,
"CELERYD_CONSUMER": "celery.worker.consumer:Consumer",
"CELERY_INCLUDE": [],
"CELERYD_MAX_TASKS_PER_CHILD": null,
"CELERYD_LOG_FORMAT": "[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
"CELERY_ANNOTATIONS": null,
"CELERY_MESSAGE_COMPRESSION": null,
"CELERY_SEND_TASK_ERROR_EMAILS": false,
"CELERY_QUEUES": null,
"CELERY_ACKS_LATE": false,
"CELERYMON_LOG_FORMAT": "[%(asctime)s: %(levelname)s] %(message)s",
"BROKER_HOST": null,
"CELERY_EVENT_QUEUE_TTL": null,
"BROKER_FAILOVER_STRATEGY": null,
"CELERY_RESULT_BACKEND": "djcelery.backends.database:DatabaseBackend",
"CELERY_BROADCAST_EXCHANGE": "celeryctl",
"CELERYBEAT_LOG_FILE": null,
"CELERYD_TIMER": null,
"CELERY_RESULT_DBURI": null,
"CELERY_DEFAULT_DELIVERY_MODE": 2,
"CELERYBEAT_LOG_LEVEL": "INFO"
}
嗯,不确定这是否是一个答案,但以下是我学到的一些东西 首先,这是我的Heroku文件:
web: gunicorn my_app.wsgi
celery: celery worker -A my_app -l info -c 3 --beat -b amqp://K5r-y7B1:1xxxxxxxi44x6j6PgfOJcWUksQ-Qyw1@yellow-thistle-3.bigwig.lshift.net:10653/kbOAUITC91D
其次,Heroku实际上以多线程方式运行芹菜工人。这与我的开发设置不同。结果是在dev作业上一个接一个地运行,但在Heroku上,长作业不会阻止短作业运行。基本上,Heroku将应用程序的多个实例加载到内存中,而在dev上一次只加载一个实例
第三,确切地说,这可能不是内存泄漏。据我所知,在内存使用超过400%之前,不会发生任何不好的事情。无论如何,我的应用程序已经运行了几个月,并且不断出现此警告,但没有任何负面影响。有一个名为
objgraph
的芹菜检查命令,可以生成用于内存审计的图表。您需要先pip安装objgraph
,然后运行:
celery -A Project inspect objgraph
有相同的问题。我真的很想知道发生了什么或者如何找到导致这种情况的原因。