Spring batch Spring批处理-OutOfMemoryError

我使用SpringBatch将大量数据从数据库提取到CSV文件

该程序在数据量减少的测试环境中运行良好，但当我尝试在生产中使用更大的数据量（600万条记录）运行批处理时，由于内存不足，该程序将继续运行。通过将受影响的RAM从4 GB更改为7 GB，程序将在几分钟内结束。但我不想在RAM之后运行，特别是因为数据量正在显著增加。你能帮我设置一个配置来优化内存使用并在合理的时间内完成批处理吗。 提前感谢您的帮助

job.xml

<beans xmlns:b="http://www.springframework.org/schema/batch"
  xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://www.springframework.org/schema/batch 
                        http://www.springframework.org/schema/batch/spring-batch.xsd
                        http://www.springframework.org/schema/beans 
                        http://www.springframework.org/schema/beans/spring-beans.xsd">
  <import resource="launch-context.xml" />

  <!-- increment automatique de l'id du job -->
  <bean id="idIncrementer"
    class="org.springframework.batch.core.launch.support.RunIdIncrementer" />

  <!-- Listener -->
  <bean id="extractMensJobExecutionListener" class="cnav.gipur.pci.batch.job.ExtractMensJobExecutionListener" />
<!--  <bean id="assureWriterListener" class="cnav.gipur.pci.batch.job.AssureWriterListener"/>-->

  <!--Reader-->
  <bean id="assureReader"
        class="org.springframework.batch.item.database.JdbcPagingItemReader"
        scope="step">
    <property name="dataSource" ref="dataSource" />
    <property name="queryProvider">
      <bean class="org.springframework.batch.item.database.support.PostgresPagingQueryProvider">
        <property name="selectClause" value="${query.sql.select.clause}" />
        <property name="fromClause" value="${query.sql.from.clause}" />
        <property name="whereClause" value="${query.sql.where.clause}"/>
        <property name="groupClause" value="${query.sql.group.by.clause}"/>
        <property name="sortKeys">
          <map>
            <entry key="${query.sql.sort.key.column}" value="${query.sql.sort.key.type}"/>
          </map>
        </property>
      </bean>
    </property>
    <property name="pageSize" value="10000" />
    <property name="rowMapper">
      <bean class="cnav.gipur.pci.batch.reader.AssureMapper" />
    </property>
  </bean>

  <!--beans helper s for writer-->
  <bean id="fastDateFormat" class="org.apache.logging.log4j.core.util.datetime.FastDateFormat"
        factory-method="getInstance">
    <constructor-arg value="yyyyMMdd" />
  </bean>
  <bean id="currentDate" factory-bean="fastDateFormat"
        factory-method="format">
    <constructor-arg>
      <bean class="java.util.Date" />
    </constructor-arg>
  </bean>

  <!--Writer-->
  <bean id="assureWriter" class="org.springframework.batch.item.file.FlatFileItemWriter"
  scope = "step">
    <property name="resource" ref = "outputResource"/>
    <property name="headerCallback" ref="assureHeaderCallback"/>
    <property name="lineAggregator">
      <bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator">
        <property name="delimiter" value=";"/>
        <property name="fieldExtractor">
          <bean class="cnav.gipur.pci.batch.writer.AssureFieldExtractor" />
        </property>
      </bean>
    </property>
  </bean>

  <bean id="outputResource" class="org.springframework.core.io.FileSystemResource" >
    <constructor-arg value = "./ficout/#{currentDate}_Extraction_Comptes_PCI.csv"/>
  </bean>

  <bean id="assureHeaderCallback" class="cnav.gipur.pci.batch.writer.AssureHeaderCallback" />

  <!-- job -->

  <b:job id="extractionMensJob" incrementer="idIncrementer">
    <b:listeners>
      <b:listener ref="extractMensJobExecutionListener" />
    </b:listeners>
    <b:step id="readWrite">
      <b:tasklet transaction-manager="transactionManager">
        <b:chunk reader="assureReader" writer="assureWriter" commit-interval=10000 />

      </b:tasklet>
<!--
      <b:listeners>
        <b:listener ref="assureWriterListener" />
      </b:listeners>
-->
    </b:step>
  </b:job>
</beans>

lauch-context.xml

<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

       xsi:schemaLocation="http://www.springframework.org/schema/beans
        http://www.springframework.org/schema/beans/spring-beans.xsd">

  <bean id="dataSource" class="org.apache.commons.dbcp2.BasicDataSource">
    <property name="driverClassName" value="${batch.jdbc.driver}" />
    <property name="url" value="${batch.jdbc.url}" />
    <property name="username" value="${batch.jdbc.user}" />
    <property name="password" value="${batch.jdbc.password}" />
  </bean>

  <bean id="sessionFactory"
        class="org.springframework.orm.hibernate5.LocalSessionFactoryBean">
    <property name="dataSource" ref="dataSource" />
    <property name="hibernateProperties">
      <props>
        <prop key="hibernate.hbm2ddl.auto">${hibernate.hbm2ddl.auto}</prop>
        <prop key="hibernate.dialect">${hibernate.dialect}</prop>
      </props>
    </property>
    <property name="packagesToScan" value="cnav.gipur.pci.batch.model"/>
    <property name="annotatedClasses">
      <list>
        <value>cnav.gipur.pci.batch.model.Assure</value>
        <value>cnav.gipur.pci.batch.model.Affiliation</value>
      </list>
    </property>
  </bean>


  <bean id="transactionManager" class="org.springframework.orm.hibernate5.HibernateTransactionManager" >
      <property name="sessionFactory" ref="sessionFactory"/>
      <property name="dataSource" ref="dataSource"/>
  </bean>

  <bean id="repositoryTransactionManager" class="org.springframework.batch.support.transaction.ResourcelessTransactionManager"/>


  <bean id="jobRepository"
        class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean"
        p:transactionManager-ref="repositoryTransactionManager" />

  <bean id="jobOperator"
        class="org.springframework.batch.core.launch.support.SimpleJobOperator"
        p:jobLauncher-ref="simpleJobLauncher" p:jobExplorer-ref="jobExplorer"
        p:jobRepository-ref="jobRepository" p:jobRegistry-ref="jobRegistry" />

  <bean id="jobExplorer"
        class="org.springframework.batch.core.explore.support.JobExplorerFactoryBean"
        p:dataSource-ref="dataSource" />

  <bean id="jobRegistry"
        class="org.springframework.batch.core.configuration.support.MapJobRegistry" />

  <bean class="org.springframework.batch.core.configuration.support.JobRegistryBeanPostProcessor">
    <property name="jobRegistry" ref="jobRegistry" />
  </bean>

  <bean id="simpleJobLauncher"
        class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
    <property name="jobRepository" ref="jobRepository" />
  </bean>

  <bean id="placeholderProperties" class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
    <property name="location" value="classpath:batch-default.properties"/>
  </bean>

</beans>

${hibernate.hbm2ddl.auto}
${hibernate.dial}
cnav.gipur.pci.batch.model.Assure
cnav.gipur.pci.batch.model.Affiliation

春季批量版本：
4.4.2

以下是一些实施细节：

读取器读取数据库中的数据。处理器中没有处理，因为所有处理都是通过SQL查询完成的。writer将结果导出到CSV文件

，可能类似于与Mysql相关的CSV文件，但我看到您使用的是Postgres。在postgres中实现流媒体应该满足以下几个条件：。确保您满足这些条件（使用postgres 7.4+，使用协议v3+，等等）。