如何为spring批处理数据和业务数据分别配置java数据源?我应该做吗?
我的主要工作只是读取操作,而另一个则在如何为spring批处理数据和业务数据分别配置java数据源?我应该做吗?,spring,spring-batch,spring-boot,spring-java-config,Spring,Spring Batch,Spring Boot,Spring Java Config,我的主要工作只是读取操作,而另一个则在MyISAM引擎上执行一些写入操作,该引擎忽略事务,因此我不需要必要的事务支持。如何配置Spring批处理,使JobRepository拥有自己的数据源,与保存业务数据的数据源分开?初始的one数据源配置如下所示: @Configuration public class StandaloneInfrastructureConfiguration { @Autowired Environment env; @Bean pub
MyISAM引擎上执行一些写入操作,该引擎忽略事务,因此我不需要必要的事务支持。如何配置Spring批处理
,使JobRepository
拥有自己的数据源,与保存业务数据的数据源分开?初始的one数据源配置如下所示:
@Configuration
public class StandaloneInfrastructureConfiguration {
@Autowired
Environment env;
@Bean
public LocalContainerEntityManagerFactoryBean entityManagerFactory() {
LocalContainerEntityManagerFactoryBean em = new LocalContainerEntityManagerFactoryBean();
em.setDataSource(dataSource());
em.setPackagesToScan(new String[] { "org.podcastpedia.batch.*" });
JpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter();
em.setJpaVendorAdapter(vendorAdapter);
em.setJpaProperties(additionalJpaProperties());
return em;
}
Properties additionalJpaProperties() {
Properties properties = new Properties();
properties.setProperty("hibernate.hbm2ddl.auto", "none");
properties.setProperty("hibernate.dialect", "org.hibernate.dialect.MySQL5Dialect");
properties.setProperty("hibernate.show_sql", "true");
return properties;
}
@Bean
public DataSource dataSource(){
return DataSourceBuilder.create()
.url(env.getProperty("db.url"))
.driverClassName(env.getProperty("db.driver"))
.username(env.getProperty("db.username"))
.password(env.getProperty("db.password"))
.build();
}
@Bean
public PlatformTransactionManager transactionManager(EntityManagerFactory emf){
JpaTransactionManager transactionManager = new JpaTransactionManager();
transactionManager.setEntityManagerFactory(emf);
return transactionManager;
}
}
然后它被导入到作业
的配置类中,@EnableBatchProcessing
注释自动使用它。我最初的想法是尝试设置配置类来扩展DefaultBatchConfigurer
,但后来我得到了一个
BeanCurrentlyIncremationException(org.springframework.beans.factory.BeanCurrentlyIncremationException:创建名为jobBuilders的bean时出错:请求的bean当前正在创建中:是否存在无法解决的循环引用?)
@配置
@启用批处理
@导入({StandaloneInfrastructureConfiguration.class,NotifySubscribersServicesConfiguration.class})
公共类NotifySubscribersJobConfiguration扩展了DefaultBatchConfigurer{
@自动连线
私人建筑工人工厂建筑工人;
@自动连线
私人StepBuilder工厂stepBuilders;
@自动连线
私有数据源;
@自动连线
环境环境;
@凌驾
@自动连线
public void setDataSource(javax.sql.DataSource数据源){
super.setDataSource(batchDataSource());
}
专用数据源batchDataSource(){
返回DataSourceBuilder.create()
.url(env.getProperty(“batchdb.url”))
.driverClassName(env.getProperty(“batchdb.driver”))
.username(env.getProperty(“batchdb.username”))
.password(env.getProperty(“batchdb.password”))
.build();
}
@豆子
public ItemReader notifySubscribersReader(){
JdbcCursorItemReader=新的JdbcCursorItemReader();
String sql=“select*from users where is\u email\u subscriber not null”;
reader.setSql(sql);
reader.setDataSource(数据源);
setRowMapper(rowMapper());
返回读取器;
}
........
}
任何想法都是非常受欢迎的。该项目可在GitHub-
非常感谢。你已经试过类似的东西了吗
@Bean(name="batchDataSource")
public DataSource batchDataSource(){
return DataSourceBuilder.create()
.url(env.getProperty("batchdb.url"))
.driverClassName(env.getProperty("batchdb.driver"))
.username(env.getProperty("batchdb.username"))
.password(env.getProperty("batchdb.password"))
.build();
}
然后用@Primary标记另一个数据源,并在批处理配置中使用@Qualifier来指定要对batchDataSource bean进行auotwire操作。好的,这很奇怪,但它可以工作。将数据源移动到它自己的配置类工作得很好,并且可以自动连接
该示例是多数据源版本的:
数据源配置:
public class DataSourceConfiguration {
@Value("classpath:schema-mysql.sql")
private Resource schemaScript;
@Bean
@Primary
public DataSource hsqldbDataSource() throws SQLException {
final SimpleDriverDataSource dataSource = new SimpleDriverDataSource();
dataSource.setDriver(new org.hsqldb.jdbcDriver());
dataSource.setUrl("jdbc:hsqldb:mem:mydb");
dataSource.setUsername("sa");
dataSource.setPassword("");
return dataSource;
}
@Bean
public JdbcTemplate jdbcTemplate(final DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
@Bean
public DataSource mysqlDataSource() throws SQLException {
final SimpleDriverDataSource dataSource = new SimpleDriverDataSource();
dataSource.setDriver(new com.mysql.jdbc.Driver());
dataSource.setUrl("jdbc:mysql://localhost/spring_batch_example");
dataSource.setUsername("test");
dataSource.setPassword("test");
DatabasePopulatorUtils.execute(databasePopulator(), dataSource);
return dataSource;
}
@Bean
public JdbcTemplate mysqlJdbcTemplate(@Qualifier("mysqlDataSource") final DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
private DatabasePopulator databasePopulator() {
final ResourceDatabasePopulator populator = new ResourceDatabasePopulator();
populator.addScript(schemaScript);
return populator;
}
}
@Configuration
@EnableBatchProcessing
@Import({ DataSourceConfiguration.class, MBeanExporterConfig.class })
public class BatchConfiguration {
@Autowired
private JobBuilderFactory jobs;
@Autowired
private StepBuilderFactory steps;
@Bean
public ItemReader<Person> reader() {
final FlatFileItemReader<Person> reader = new FlatFileItemReader<Person>();
reader.setResource(new ClassPathResource("sample-data.csv"));
reader.setLineMapper(new DefaultLineMapper<Person>() {
{
setLineTokenizer(new DelimitedLineTokenizer() {
{
setNames(new String[] { "firstName", "lastName" });
}
});
setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {
{
setTargetType(Person.class);
}
});
}
});
return reader;
}
@Bean
public ItemProcessor<Person, Person> processor() {
return new PersonItemProcessor();
}
@Bean
public ItemWriter<Person> writer(@Qualifier("mysqlDataSource") final DataSource dataSource) {
final JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<Person>();
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
writer.setSql("INSERT INTO people (first_name, last_name) VALUES (:firstName, :lastName)");
writer.setDataSource(dataSource);
return writer;
}
@Bean
public Job importUserJob(final Step s1) {
return jobs.get("importUserJob").incrementer(new RunIdIncrementer()).flow(s1).end().build();
}
@Bean
public Step step1(final ItemReader<Person> reader,
final ItemWriter<Person> writer, final ItemProcessor<Person, Person> processor) {
return steps.get("step1")
.<Person, Person> chunk(1)
.reader(reader)
.processor(processor)
.writer(writer)
.build();
}
}
批量配置:
public class DataSourceConfiguration {
@Value("classpath:schema-mysql.sql")
private Resource schemaScript;
@Bean
@Primary
public DataSource hsqldbDataSource() throws SQLException {
final SimpleDriverDataSource dataSource = new SimpleDriverDataSource();
dataSource.setDriver(new org.hsqldb.jdbcDriver());
dataSource.setUrl("jdbc:hsqldb:mem:mydb");
dataSource.setUsername("sa");
dataSource.setPassword("");
return dataSource;
}
@Bean
public JdbcTemplate jdbcTemplate(final DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
@Bean
public DataSource mysqlDataSource() throws SQLException {
final SimpleDriverDataSource dataSource = new SimpleDriverDataSource();
dataSource.setDriver(new com.mysql.jdbc.Driver());
dataSource.setUrl("jdbc:mysql://localhost/spring_batch_example");
dataSource.setUsername("test");
dataSource.setPassword("test");
DatabasePopulatorUtils.execute(databasePopulator(), dataSource);
return dataSource;
}
@Bean
public JdbcTemplate mysqlJdbcTemplate(@Qualifier("mysqlDataSource") final DataSource dataSource) {
return new JdbcTemplate(dataSource);
}
private DatabasePopulator databasePopulator() {
final ResourceDatabasePopulator populator = new ResourceDatabasePopulator();
populator.addScript(schemaScript);
return populator;
}
}
@Configuration
@EnableBatchProcessing
@Import({ DataSourceConfiguration.class, MBeanExporterConfig.class })
public class BatchConfiguration {
@Autowired
private JobBuilderFactory jobs;
@Autowired
private StepBuilderFactory steps;
@Bean
public ItemReader<Person> reader() {
final FlatFileItemReader<Person> reader = new FlatFileItemReader<Person>();
reader.setResource(new ClassPathResource("sample-data.csv"));
reader.setLineMapper(new DefaultLineMapper<Person>() {
{
setLineTokenizer(new DelimitedLineTokenizer() {
{
setNames(new String[] { "firstName", "lastName" });
}
});
setFieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {
{
setTargetType(Person.class);
}
});
}
});
return reader;
}
@Bean
public ItemProcessor<Person, Person> processor() {
return new PersonItemProcessor();
}
@Bean
public ItemWriter<Person> writer(@Qualifier("mysqlDataSource") final DataSource dataSource) {
final JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<Person>();
writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
writer.setSql("INSERT INTO people (first_name, last_name) VALUES (:firstName, :lastName)");
writer.setDataSource(dataSource);
return writer;
}
@Bean
public Job importUserJob(final Step s1) {
return jobs.get("importUserJob").incrementer(new RunIdIncrementer()).flow(s1).end().build();
}
@Bean
public Step step1(final ItemReader<Person> reader,
final ItemWriter<Person> writer, final ItemProcessor<Person, Person> processor) {
return steps.get("step1")
.<Person, Person> chunk(1)
.reader(reader)
.processor(processor)
.writer(writer)
.build();
}
}
@配置
@启用批处理
@导入({DataSourceConfiguration.class,MBeanExporterConfig.class})
公共类批处理配置{
@自动连线
私人建筑工地;
@自动连线
私人StepBuilderFactorySteps;
@豆子
公共项目阅读器(){
最终FlatFileItemReader=新FlatFileItemReader();
setResource(新类路径资源(“sample data.csv”);
reader.setLineMapper(新的DefaultLineMapper(){
{
setLineTokenizer(新的DelimitedLineTokenizer(){
{
setNames(新字符串[]{“firstName”,“lastName”});
}
});
setFieldSetMapper(新的BeanRapperFieldSetMapper(){
{
setTargetType(Person.class);
}
});
}
});
返回读取器;
}
@豆子
公共项目处理器(){
返回新的PersonItemProcessor();
}
@豆子
公共ItemWriter编写器(@Qualifier(“mysqlDataSource”)最终数据源数据源){
最终JdbcBatchItemWriter编写器=新的JdbcBatchItemWriter();
writer.setItemSqlParameterSourceProvider(新的BeanPropertyItemSqlParameterSourceProvider());
setSql(“插入人(名字、姓氏)值(:firstName,:lastName)”;
writer.setDataSource(dataSource);
返回作者;
}
@豆子
公共作业导入作业(最后一步s1){
return jobs.get(“importUserJob”).incrementer(new RunIdIncrementer()).flow(s1.end().build();
}
@豆子
公共步骤步骤1(最终项目阅读器,
最终项目编写器、最终项目处理器){
返回步骤。获取(“步骤1”)
.chunk(1)
.读卡器(读卡器)
.处理器(处理器)
.作者(作者)
.build();
}
}
我的数据源位于单独的配置类中。在批处理配置中,我们扩展DefaultBatchConfigurer并重写setDataSource方法,通过@Qualifier传入特定数据库以与SpringBatch一起使用。我无法使用构造函数版本使其工作,但setter方法对我有效
我的Reader、Processor和Writer都在它们自己的自包含类中,以及步骤
这是使用Spring Boot 1.1.8和Spring Batch 3.0.1注意:我们对使用Spring Boot 1.1.5的项目进行了不同的设置,但在较新的版本上不起同样的作用
package org.sample.config.jdbc;
import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.env.Environment;
import com.atomikos.jdbc.AtomikosDataSourceBean;
import com.mysql.jdbc.jdbc2.optional.MysqlXADataSource;
/**
* The Class DataSourceConfiguration.
*
*/
@Configuration
public class DataSourceConfig {
private final static Logger log = LoggerFactory.getLogger(DataSourceConfig.class);
@Autowired private Environment env;
/**
* Siphon data source.
*
* @return the data source
*/
@Bean(name = "mainDataSource")
@Primary
public DataSource mainDataSource() {
final String user = this.env.getProperty("db.main.username");
final String password = this.env.getProperty("db.main.password");
final String url = this.env.getProperty("db.main.url");
return this.getMysqlXADataSource(url, user, password);
}
/**
* Batch data source.
*
* @return the data source
*/
@Bean(name = "batchDataSource", initMethod = "init", destroyMethod = "close")
public DataSource batchDataSource() {
final String user = this.env.getProperty("db.batch.username");
final String password = this.env.getProperty("db.batch.password");
final String url = this.env.getProperty("db.batch.url");
return this.getAtomikosDataSource("metaDataSource", this.getMysqlXADataSource(url, user, password));
}
/**
* Gets the mysql xa data source.
*
* @param url the url
* @param user the user
* @param password the password
* @return the mysql xa data source
*/
private MysqlXADataSource getMysqlXADataSource(final String url, final String user, final String password) {
final MysqlXADataSource mysql = new MysqlXADataSource();
mysql.setUser(user);
mysql.setPassword(password);
mysql.setUrl(url);
mysql.setPinGlobalTxToPhysicalConnection(true);
return mysql;
}
/**
* Gets the atomikos data source.
*
* @param resourceName the resource name
* @param xaDataSource the xa data source
* @return the atomikos data source
*/
private AtomikosDataSourceBean getAtomikosDataSource(final String resourceName, final MysqlXADataSource xaDataSource) {
final AtomikosDataSourceBean atomikos = new AtomikosDataSourceBean();
atomikos.setUniqueResourceName(resourceName);
atomikos.setXaDataSource(xaDataSource);
atomikos.setMaxLifetime(3600);
atomikos.setMinPoolSize(2);
atomikos.setMaxPoolSize(10);
return atomikos;
}
}
package org.sample.settlement.batch;
import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.DefaultBatchConfigurer;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.transaction.PlatformTransactionManager;
/**
* The Class BatchConfiguration.
*
*/
@Configuration
@EnableBatchProcessing
public class BatchConfiguration extends DefaultBatchConfigurer {
private final static Logger log = LoggerFactory.getLogger(BatchConfiguration.class);
@Autowired private JobBuilderFactory jobs;
@Autowired private StepBuilderFactory steps;
@Autowired private PlatformTransactionManager transactionManager;
@Autowired @Qualifier("processStep") private Step processStep;
/**
* Process payments job.
*
* @return the job
*/
@Bean(name = "processJob")
public Job processJob() {
return this.jobs.get("processJob")
.incrementer(new RunIdIncrementer())
.start(processStep)
.build();
}
@Override
@Autowired
public void setDataSource(@Qualifier("batchDataSource") DataSource batchDataSource) {
super.setDataSource(batchDataSource);
}
}
假设您有两个数据源,一个用于spring批处理元数据,如作业详细信息[假设CONFIGDB],另一个用于业务数据[假设AppDB]:
将CONFIGDB注入jobRepository,如下所示:
<bean id="jobRepository"
class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
<property name="transactionManager" ref="transactionManager" />
<property name="dataSource" ref="CONFIGDB" />
<property name="databaseType" value="db2" />
<property name="tablePrefix" value="CONFIGDB.BATCH_" />
</bean>
}根据:
在应用程序属性中,可以使用常规数据源属性:
app.datasource.first.type=com.zaxxer.hikari.HikariDataSource
app.datasource.first.maximum-pool-size=30
app.datasource.second.url=jdbc:mysql://localhost/test
app.datasource.second.username=dbuser
app.datasource.second.password=dbpass
app.datasource.second.max-total=30
正如冻结在两个数据源中所建议的那样:
@Bean
@Primary
@ConfigurationProperties("app.datasource.first")
public DataSourceProperties firstDataSourceProperties() {
return new DataSourceProperties();
}
@Bean
@Primary
@ConfigurationProperties("app.datasource.first")
public DataSource firstDataSource() {
return firstDataSourceProperties().initializeDataSourceBuilder().build();
}
@Bean
@ConfigurationProperties("app.datasource.second")
public DataSourceProperties secondDataSourceProperties() {
return new DataSourceProperties();
}
@Bean
@ConfigurationProperties("app.datasource.second")
public DataSource secondDataSource() {
return secondDataSourceProperties().initializeDataSourceBuilder().build();
}
app.datasource.first.type=com.zaxxer.hikari.HikariDataSource
app.datasource.first.maximum-pool-size=30
app.datasource.second.url=jdbc:mysql://localhost/test
app.datasource.second.username=dbuser
app.datasource.second.password=dbpass
app.datasource.second.max-total=30
# Sample configuraion using a H2 in-memory DB
domain.datasource.jdbcUrl=jdbc:h2:mem:domain-ds;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
domain.datasource.username=sa
domain.datasource.password=
domain.datasource.driver=org.h2.Driver
batch.datasource.jdbcUrl=jdbc:h2:mem:batch-ds;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE
batch.datasource.username=sa
batch.datasource.password=
batch.datasource.driver=org.h2.Driver
/**
* Qualifier annotation for a DataSource to be injected into Batch auto-configuration. Can
* be used on a secondary data source, if there is another one marked as
* {@link Primary @Primary}.
*
* @author Dmytro Nosan
* @since 2.2.0
*/
@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.TYPE, ElementType.ANNOTATION_TYPE })
@Retention(RetentionPolicy.RUNTIME)
@Documented
@Qualifier
public @interface BatchDataSource {
}
@BatchDataSource
@Bean("batchDataSource")
public DataSource batchDataSource(@Qualifier("batchDataSourceProperties") DataSourceProperties dataSourceProperties) {
return dataSourceProperties
.initializeDataSourceBuilder()
.type(HikariDataSource.class)
.build();
}