Netty &引用；OutOfMemoryError:直接缓冲存储器“；当运行风暴0.9.6时

我在storm0.9.6上运行了一个简单的wordcount拓扑，发现了2个异常行为

Netty重新连接了很多次

直接内存不断增长。最后抛出java.lang.OutOfMemoryError

以下是意外信息：

2015-12-15T21:30:18.546+0800 b.s.d.executor [INFO] Activating spout emitSpout:(1)
2015-12-15T21:30:18.578+0800 b.s.d.executor [INFO] Preparing bolt __system:(-1)
2015-12-15T21:30:18.614+0800 b.s.d.executor [INFO] Prepared bolt __system:(-1)
2015-12-15T21:38:37.453+0800 b.s.m.n.Client [ERROR] failed to send 943 messages to Netty-Client-eda.local/192.168.137.2:6708: java.lang.OutOfMemoryError: Direct buffer memory
2015-12-15T21:38:37.457+0800 b.s.m.n.Client [ERROR] failed to send 944 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.464+0800 b.s.m.n.Client [ERROR] failed to send 945 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.472+0800 b.s.m.n.Client [ERROR] failed to send 944 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.477+0800 b.s.m.n.Client [ERROR] failed to send 943 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.479+0800 b.s.m.n.Client [ERROR] failed to send 938 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.481+0800 b.s.m.n.Client [ERROR] failed to send 946 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.482+0800 b.s.m.n.Client [ERROR] failed to send 943 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.484+0800 b.s.m.n.Client [ERROR] failed to send 943 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.485+0800 b.s.m.n.Client [ERROR] failed to send 941 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.490+0800 b.s.m.n.Client [ERROR] failed to send 942 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.504+0800 b.s.m.n.Client [ERROR] failed to send 941 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.505+0800 b.s.m.n.Client [ERROR] failed to send 943 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.507+0800 b.s.m.n.Client [ERROR] failed to send 944 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.510+0800 b.s.m.n.Client [ERROR] failed to send 944 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.516+0800 b.s.m.n.StormClientHandler [INFO] Connection failed Netty-Client-eda.local/192.168.137.2:6708
java.lang.OutOfMemoryError: Direct buffer memory
    at java.nio.Bits.reserveMemory(Bits.java:658) ~[na:1.7.0_67]
    at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123) ~[na:1.7.0_67]
    at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:306) ~[na:1.7.0_67]
    at org.apache.storm.netty.buffer.ChannelBuffers.directBuffer(ChannelBuffers.java:167) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.buffer.ChannelBuffers.directBuffer(ChannelBuffers.java:151) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.messaging.netty.MessageBatch.buffer(MessageBatch.java:84) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.messaging.netty.MessageEncoder.encode(MessageEncoder.java:32) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.handler.codec.oneone.OneToOneEncoder.doEncode(OneToOneEncoder.java:66) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:59) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:591) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:582) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.channel.Channels.write(Channels.java:704) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.channel.Channels.write(Channels.java:671) ~[storm-core-0.9.6.jar:0.9.6]
    at org.apache.storm.netty.channel.AbstractChannel.write(AbstractChannel.java:248) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.messaging.netty.Client.flushMessages(Client.java:323) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.messaging.netty.Client.send(Client.java:250) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.utils.TransferDrainer.send(TransferDrainer.java:54) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.daemon.worker$mk_transfer_tuples_handler$fn__4610$fn__4611.invoke(worker.clj:338) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.daemon.worker$mk_transfer_tuples_handler$fn__4610.invoke(worker.clj:336) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.disruptor$clojure_handler$reify__1445.onEvent(disruptor.clj:58) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.utils.DisruptorQueue.consumeBatchToCursor(DisruptorQueue.java:132) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.utils.DisruptorQueue.consumeBatchWhenAvailable(DisruptorQueue.java:106) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.disruptor$consume_batch_when_available.invoke(disruptor.clj:80) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.disruptor$consume_loop_STAR_$fn__1458.invoke(disruptor.clj:94) ~[storm-core-0.9.6.jar:0.9.6]
    at backtype.storm.util$async_loop$fn__460.invoke(util.clj:463) ~[storm-core-0.9.6.jar:0.9.6]
    at clojure.lang.AFn.run(AFn.java:24) ~[clojure-1.5.1.jar:na]
    at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
2015-12-15T21:38:37.516+0800 b.s.m.n.Client [ERROR] failed to send 945 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException
2015-12-15T21:38:37.521+0800 b.s.m.n.Client [ERROR] failed to send 944 messages to Netty-Client-eda.local/192.168.137.2:6708: java.nio.channels.ClosedChannelException

---

有人知道如何解决这两个问题吗？

关于不断增长的内存：看起来您在拓扑结构中有一个瓶颈（即消耗操作员的小dop），并且您不使用acking进行流量控制。因此，要么尝试增加并行度，要么使用acking进行流控制（参数

MAX\u SPOUT\u PENDING

）。对于内蒂：你所说的“连接这么多次”是什么意思？例如，在启动时，需要一段时间才能建立连接，因为所有工作节点都必须首先启动。当您运行拓扑时，我不确定是否真的存在问题——现在我对此表示怀疑。你有关于它的错误日志吗？关于不断增长的内存，我增加了瓶颈操作符的并行性，它可以工作。对于风暴来袭时netty重新连接的问题，它似乎是netty的一个bug。我增加了storm.messaging.netty.max\u wait\u ms的价值，情况有所改善。太好了！不确定这种烦人的重新连接是一个bug还是没有被考虑。建立网络连接只需要一些时间。提交拓扑时，新的工作JVM将在不同的机器上启动，这当然是一个异步操作。因此，如果第一个JVM已启动并试图通过Netty连接到另一个工作JVM，则此工作JVM可能尚未准备就绪，因此在第二个工作JVM准备好接受网络请求之前，您将看到这些日志消息。谢谢您的回复！

#topology.skip.missing.kryo.registrations: false
storm.messaging.netty.flush.check.interval.ms: 100
storm.messaging.transport: "backtype.storm.messaging.netty.Context"
storm.messaging.netty.server_worker_threads: 1
storm.messaging.netty.client_worker_threads: 1
#storm.messaging.netty.buffer_size: 5242880
storm.messaging.netty.transfer.batch.size: 462144
storm.messaging.netty.buffer_size: 5242880
storm.messaging.netty.max_retries: 100
storm.messaging.netty.max_wait_ms: 1000
storm.messaging.netty.min_wait_ms: 100
topology.backpressure.enable: false
#storm.scheduler: "backtype.storm.scheduler.DefaultScheduler"
worker.childopts: "-Xmx768m -XX:NewSize=128m -XX:MaxNewSize=128m -XX:PermSize=64m -XX:MaxPermSize=64m -XX:MaxDirectMemorySize=128m -Djava.net.preferIPv4Stack=true  -XX:+UseParNewGC  -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -verbose:gc -Xloggc:/ffcs/gc/gc-storm-worker-%ID%.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/ffcs/gc/heapdump"
#worker.childopts: "-Xmx1024m"
supervisor.childopts: "-Xmx256m"
nimbus.childopts: "-Xmx768m"
ui.childopts: "-Xmx768m"
nimbus.thrift.threads: 256