Hibernate search Infinispan中的复制超时

我正在使用一个10节点的Infinispan集群作为Hibernate搜索后端。我们的服务器在Java 1.6_24上运行TC server 2.5（tomcat 6.0.32）。我们使用jGroups 2.12.1.3来处理来自每个节点的集群缓存写入，以及多播UDP传输

当我们在集群中启动3+个节点时，最终其中一个节点开始记录复制超时。无论是将Infinispan配置为复制模式还是分布式缓存模式，我们都观察到了相同的结果。尽管集群的其余部分保持稳定，但故障节点基本上无法进行搜索

我们的配置：

英菲尼斯潘：

<?xml version="1.0" encoding="UTF-8"?>
 <infinispan
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="urn:infinispan:config:5.0 http://www.infinispan.org/schemas/infinispan-config-5.0.xsd"
    xmlns="urn:infinispan:config:5.0">
<global>
    <globalJmxStatistics
        enabled="true"
        cacheManagerName="HibernateSearch"
        allowDuplicateDomains="true" />
    <transport
        clusterName="HibernateSearch-Infinispan-cluster-MT"
        distributedSyncTimeout="50000">
        <properties>
            <property name="configurationFile" value="infinispan-udp.cfg.xml" />
        </properties>
    </transport>
    <shutdown
        hookBehavior="DONT_REGISTER" />
</global>


<default>
    <locking
        lockAcquisitionTimeout="20000"
        writeSkewCheck="false"
        concurrencyLevel="5000"
        useLockStriping="false" />
    <storeAsBinary storeKeysAsBinary="false" storeValuesAsBinary="true"
        enabled="false" />
    <invocationBatching
        enabled="true" />
    <clustering
        mode="replication">
        <stateRetrieval
            timeout="60000"
            logFlushTimeout="65000"
            fetchInMemoryState="true"
            alwaysProvideInMemoryState="true" />
        <sync
            replTimeout="50000" />
        <l1 enabled="false" />
    </clustering>
    <jmxStatistics
        enabled="true" />
    <eviction
        maxEntries="-1"
        strategy="NONE" />
    <expiration
        maxIdle="-1" />
</default>


<namedCache
    name="LuceneIndexesMetadata">
    <clustering
        mode="replication">
        <stateRetrieval
            fetchInMemoryState="true"
            logFlushTimeout="30000" />
        <sync
            replTimeout="50000" />
        <l1 enabled="false" />
    </clustering>
    <locking
        lockAcquisitionTimeout="20000"
        writeSkewCheck="false"
        concurrencyLevel="5000"
        useLockStriping="false" />
    <loaders shared="true" preload="true">
        <loader class="org.infinispan.loaders.jdbm.JdbmCacheStore" fetchPersistentState="false" ignoreModifications="false" purgeOnStartup="false">
            <properties>
                <property name="location" value="/usr/local/tc/.index/metadata" />
            </properties>
        </loader>
    </loaders>
</namedCache>
<namedCache
    name="LuceneIndexesData">
    <clustering
        mode="replication">
        <stateRetrieval
            fetchInMemoryState="true"
            logFlushTimeout="30000" />
        <sync
            replTimeout="50000" />
        <l1 enabled="false" />
    </clustering>
    <locking
        lockAcquisitionTimeout="20000"
        writeSkewCheck="false"
        concurrencyLevel="5000"
        useLockStriping="false" />
    <loaders shared="true" preload="true">
        <loader class="org.infinispan.loaders.jdbm.JdbmCacheStore" fetchPersistentState="false" ignoreModifications="false" purgeOnStartup="false">
            <properties>
                <property name="location" value="/usr/local/tc/.index/data" />
            </properties>
        </loader>
    </loaders>
</namedCache>
<namedCache
    name="LuceneIndexesLocking">
    <clustering
        mode="replication">
        <stateRetrieval
            fetchInMemoryState="true"
            logFlushTimeout="30000" />
        <sync
            replTimeout="50000" />
        <l1 enabled="false" />
    </clustering>
    <locking
        lockAcquisitionTimeout="20000"
        writeSkewCheck="false"
        concurrencyLevel="5000"
        useLockStriping="false" />
</namedCache>

jgroup（UDP）：

以及我们观察到的错误：

10-31-2011 13:53:02 ERROR Hibernate Search: Directory writer-3 interceptors.InvocationContextInterceptor: ISPN000136: Execution error
org.infinispan.util.concurrent.TimeoutException: Replication timeout for tc-cluster-0105-21082
          at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:71)
          at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:452)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:132)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:156)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:265)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:252)
          at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:235)
          at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:228)
          at org.infinispan.interceptors.ReplicationInterceptor.handleCrudMethod(ReplicationInterceptor.java:116)
          at org.infinispan.interceptors.ReplicationInterceptor.visitPutKeyValueCommand(ReplicationInterceptor.java:79)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.LockingInterceptor.visitPutKeyValueCommand(LockingInterceptor.java:294)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:133)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:214)
          at org.infinispan.interceptors.TxInterceptor.visitPutKeyValueCommand(TxInterceptor.java:162)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:114)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:104)
          at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:64)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:77)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:274)
          at org.infinispan.CacheImpl.putIfAbsent(CacheImpl.java:524)
          at org.infinispan.CacheSupport.putIfAbsent(CacheSupport.java:74)
          at org.infinispan.lucene.locking.BaseLuceneLock.obtain(BaseLuceneLock.java:65)
          at org.apache.lucene.store.Lock.obtain(Lock.java:72)
          at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1097)
          at org.hibernate.search.backend.Workspace.createNewIndexWriter(Workspace.java:202)
          at org.hibernate.search.backend.Workspace.getIndexWriter(Workspace.java:180)
          at org.hibernate.search.backend.impl.lucene.PerDPQueueProcessor.run(PerDPQueueProcessor.java:103)
          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
          at java.lang.Thread.run(Thread.java:662)

10-31-2011 13:53:02错误休眠搜索：目录编写器-3侦听器。调用上下文侦听器：ISPN000136:执行错误
org.infinispan.util.concurrent.TimeoutException:tc-cluster-0105-21082的复制超时
位于org.infinispan.remoting.transport.AbstractTransport.ParseResponseAndToResponseList（AbstractTransport.java:71）
位于org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokerRemote（JGroupsTransport.java:452）
位于org.infinispan.remoting.rpc.RpcManagerImpl.invokerRemotey（RpcManagerImpl.java:132）
位于org.infinispan.remoting.rpc.RpcManagerImpl.invokerRemote（RpcManagerImpl.java:156）
位于org.infinispan.remoting.rpc.RpcManagerImpl.invokerRemotey（RpcManagerImpl.java:265）
位于org.infinispan.remoting.rpc.RpcManagerImpl.invokerRemote（RpcManagerImpl.java:252）
位于org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand（RpcManagerImpl.java:235）
位于org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand（RpcManagerImpl.java:228）
位于org.infinispan.interceptors.ReplicationInterceptor.handleCrudMethod（ReplicationInterceptor.java:116）
位于org.infinispan.interceptors.ReplicationInterceptor.visitPutKeyValueCommand（ReplicationInterceptor.java:79）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.LockingInterceptor.visitPutKeyValueCommand（LockingInterceptor.java:294）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.base.CommandInterceptor.handleDefault（CommandInterceptor.java:133）
位于org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand（AbstractVisitor.java:60）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.TxInterceptor.EnstractWriteAndInvokenext（TxInterceptor.java:214）
位于org.infinispan.interceptors.TxInterceptor.visitPutKeyValueCommand（TxInterceptor.java:162）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand（CacheMgmtInterceptor.java:114）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.InvocationContextInterceptor.handleAll（InvocationContextInterceptor.java:104）
位于org.infinispan.interceptors.InvocationContextInterceptor.handleDefault（InvocationContextInterceptor.java:64）
位于org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand（AbstractVisitor.java:60）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor（CommandInterceptor.java:119）
位于org.infinispan.interceptors.BatchingInterceptor.handleDefault（BatchingInterceptor.java:77）
位于org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand（AbstractVisitor.java:60）
位于org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor（PutKeyValueCommand.java:77）
位于org.infinispan.interceptors.InterceptorChain.invoke（InterceptorChain.java:274）
位于org.infinispan.CacheImpl.putIfAbsent（CacheImpl.java:524）
位于org.infinispan.CacheSupport.putIfAbsent（CacheSupport.java:74）
位于org.infinispan.lucene.locking.BaseLuceneLock.get（BaseLuceneLock.java:65）
位于org.apache.lucene.store.Lock.get（Lock.java:72）
位于org.apache.lucene.index.IndexWriter（IndexWriter.java:1097）
位于org.hibernate.search.backend.Workspace.createNewIndexWriter（Workspace.java:202）
位于org.hibernate.search.backend.Workspace.getIndexWriter（Workspace.java:180）
位于org.hibernate.search.backend.impl.lucene.PerDPQueueProcessor.run（PerDPQueueProcessor.java:103）
位于java.util.concurrent.ThreadPoolExecutor$Worker.runTask（ThreadPoolExecutor.java:886）
10-31-2011 13:53:02 ERROR Hibernate Search: Directory writer-3 interceptors.InvocationContextInterceptor: ISPN000136: Execution error
org.infinispan.util.concurrent.TimeoutException: Replication timeout for tc-cluster-0105-21082
          at org.infinispan.remoting.transport.AbstractTransport.parseResponseAndAddToResponseList(AbstractTransport.java:71)
          at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:452)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:132)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:156)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:265)
          at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:252)
          at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:235)
          at org.infinispan.remoting.rpc.RpcManagerImpl.broadcastRpcCommand(RpcManagerImpl.java:228)
          at org.infinispan.interceptors.ReplicationInterceptor.handleCrudMethod(ReplicationInterceptor.java:116)
          at org.infinispan.interceptors.ReplicationInterceptor.visitPutKeyValueCommand(ReplicationInterceptor.java:79)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.LockingInterceptor.visitPutKeyValueCommand(LockingInterceptor.java:294)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:133)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:214)
          at org.infinispan.interceptors.TxInterceptor.visitPutKeyValueCommand(TxInterceptor.java:162)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:114)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:104)
          at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:64)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:119)
          at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:77)
          at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:60)
          at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:77)
          at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:274)
          at org.infinispan.CacheImpl.putIfAbsent(CacheImpl.java:524)
          at org.infinispan.CacheSupport.putIfAbsent(CacheSupport.java:74)
          at org.infinispan.lucene.locking.BaseLuceneLock.obtain(BaseLuceneLock.java:65)
          at org.apache.lucene.store.Lock.obtain(Lock.java:72)
          at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1097)
          at org.hibernate.search.backend.Workspace.createNewIndexWriter(Workspace.java:202)
          at org.hibernate.search.backend.Workspace.getIndexWriter(Workspace.java:180)
          at org.hibernate.search.backend.impl.lucene.PerDPQueueProcessor.run(PerDPQueueProcessor.java:103)
          at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
          at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
          at java.lang.Thread.run(Thread.java:662)