连接拒绝使用python在elasticsearch中执行数据批量

我有一个docker compose，它包含两个容器，一个包含python，另一个包含Elasticsearch。我正在尝试在Elasticsearch数据库中批量处理大量txt文件，读取并每行插入一行，但由于某些原因，Elasticsearch机器在批量处理过程中停止，连接被拒绝

这就是我得到的错误：

Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 160, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/usr/local/lib/python3.7/site-packages/urllib3/util/connection.py", line 84, in create_connection
    raise err
  File "/usr/local/lib/python3.7/site-packages/urllib3/util/connection.py", line 74, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

这是我的docker撰写文件：

version: "3.7"

services:
  web:
    container_name: web
    restart: "always"
    build: "./web"
    hostname: web
    command: python manage.py runserver 0.0.0.0:8000
    ports:
      - 8000:8000
    depends_on:
      - els
    volumes:
      - ./web:/web
      - ./data:/data
    networks:
      - esnet

  els:
    container_name: els
    build: "./containers/elasticsearch"
    restart: "always"
    hostname: els
    ports:
      - 9200:9200
      - 9300:9300
    volumes:
      - "./data/elasticsearch:/usr/share/elasticsearch/data"
    ulimits:
      nproc: 3000
      nofile: 65536
      memlock: -1
    environment:
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx8G"
      - discovery.type=single-node
      - xpack.security.enabled=false
    networks:
      - esnet

volumes:
  web:
  data:

networks:
  esnet:

这是elasticsearch dockerfile：

FROM docker.elastic.co/elasticsearch/elasticsearch:7.2.1

ADD ./elasticsearch.yml /usr/share/elasticsearch/config

EXPOSE 9200 9300

这是配置：

http.host: 0.0.0.0

http.cors.allow-origin: "*"
http.cors.enabled: true
http.cors.allow-headers: "X-Requested-With, X-Auth-Token, Content-Type, Content-Length, Authorization"
http.cors.allow-methods: "OPTIONS, HEAD, GET, POST, PUT, DELETE"
http.cors.allow-credentials: true

这是我的数据批量代码：

bulk_list = []
with open(file) as f:
    lines = f.readlines()
for line in lines:
    l = line.strip()
    l = l.replace("\n", "")
    if l:
        bulk_list.append({"...some data..."})
helpers.bulk(els, bulk_list)

这是我的

http://localhost:9200/_nodes/thread_pool?pretty

write参数：

"write" : {
    "type" : "fixed",
    "size" : 8,
    "queue_size" : 200
}

我认为后者是问题所在，但我不太确定解决这个问题的最佳方法是什么，避免连接被拒绝

谢谢

增加：

这是docker compose中的日志错误：

{"type": "server", "timestamp": "2020-05-24T20:04:13,927Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "els", "message": "fatal error in thread [elasticsearch[els][refresh][T#1]], exiting", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,
"stacktrace": ["java.lang.InternalError: a fault occurred in a recent unsafe memory access operation in compiled Java code",
"at org.apache.lucene.index.ByteSliceReader.readByte(ByteSliceReader.java:73) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.store.DataInput.readVInt(DataInput.java:126) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FreqProxFields$FreqProxPostingsEnum.nextPosition(FreqProxFields.java:501) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:937) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:347) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225) ~[lucene-core-8.5.1.jar:8.5.1 edb9fc409398f2c3446883f9f80595c884d245d0 - ivera - 2020-04-08 08:55:42]",
"at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3234) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:835) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService.access$200(IndexService.java:109) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:967) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) ~[elasticsearch-7.7.0.jar:7.7.0]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) ~[?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) ~[?:?]",
"at java.lang.Thread.run(Thread.java:832) [?:?]"] }
{"type": "server", "timestamp": "2020-05-24T20:04:13,937Z", "level": "ERROR", "component": "o.e.i.e.Engine", "cluster.name": "elasticsearch", "node.name": "els", "message": " [fincorpus][0] already closed by tragic event on the index writer", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
fatal error in thread [elasticsearch[els][refresh][T#1]], exiting
java.lang.InternalError: a fault occurred in a recent unsafe memory access operation in compiled Java code
at org.apache.lucene.index.ByteSliceReader.readByte(ByteSliceReader.java:73)
at org.apache.lucene.store.DataInput.readVInt(DataInput.java:126)
at org.apache.lucene.index.FreqProxFields$FreqProxPostingsEnum.nextPosition(FreqProxFields.java:501)
at org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:150)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:937)
at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:347)
at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
at org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
at org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:362)
at org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
at org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1625)
at org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1604)
at org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3234)
at org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:835)
at org.elasticsearch.index.IndexService.access$200(IndexService.java:109)
at org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:967)
at org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base/java.lang.Thread.run(Thread.java:832)
{"type": "server", "timestamp": "2020-05-24T20:04:13,941Z", "level": "ERROR", "component": "o.e.b.ElasticsearchUncaughtExceptionHandler", "cluster.name": "elasticsearch", "node.name": "els", "message": "fatal error in thread [elasticsearch[els][write][T#7]], exiting", "cluster.uuid": "Ic_DMuXJTQqa4zu3wSEobw", "node.id": "12SGfI1qQkyZ-KJjF_okPA" ,

---

可能是网络问题。你能把你的docker文件寄出去吗？您是如何创建els变量的？当然@Lupanoide，我刚刚发布了它在我的案例中，ElasticSearch服务状态显示为失败，重新启动服务