Java 每个主题可以有一个卡夫卡消费者线程吗？
我们有使用SpringKafka（2.1.7）的Springboot应用程序。我们已经启用了并发，因此每个分区可以有一个使用者线程。因此，目前，如果我们有3个主题，每个主题有2个分区，那么将有2个使用者线程，如下所示： 消费者阅读1-[topic1-0、topic2-0、topic3-0]Java 每个主题可以有一个卡夫卡消费者线程吗？,java,spring-boot,apache-kafka,spring-kafka,Java,Spring Boot,Apache Kafka,Spring Kafka,我们有使用SpringKafka（2.1.7）的Springboot应用程序。我们已经启用了并发，因此每个分区可以有一个使用者线程。因此，目前，如果我们有3个主题，每个主题有2个分区，那么将有2个使用者线程，如下所示： 消费者阅读1-[topic1-0、topic2-0、topic3-0] 消费者阅读2-[主题1-1、主题2-1、主题3-1] 但是，我们不希望每个分区有一个KafkaListener（或消费者线程），而是希望每个主题有一个消费者线程。例如： 消费者阅读1-[topic1-0，to
消费者阅读2-[主题1-1、主题2-1、主题3-1]
但是，我们不希望每个分区有一个KafkaListener（或消费者线程），而是希望每个主题有一个消费者线程。例如： 消费者阅读1-[topic1-0，topic1-1]
消费者阅读2-[topic2-0，topic2-1]
消费者阅读3-[topic3-0，topic3-1]
如果不可能，即使以下设置也可以： 消费者阅读1-[topic1-0]
消费者阅读2-[topic1-1]
消费者阅读3-[topic2-0]
消费者阅读4-[topic2-1]
消费者阅读5-[topic3-0]
消费者阅读6-[topic3-1]
问题是，我们不知道手头的完整主题列表（我们使用的是通配符主题模式）。可以随时添加新主题，并且应在运行时为该新主题动态创建一个或多个新的使用者线程
有什么方法可以实现这一点吗？您可以从中为每个主题创建单独的容器，并将并发性设置为1，以便每个容器将从每个主题中使用 从2.2版开始，您可以使用同一工厂创建任何ConcurrentMessageListenerContainer。如果您想创建几个具有类似属性的容器，或者希望使用一些外部配置的工厂，例如Spring Boot auto configuration提供的工厂，那么这可能很有用。创建容器后，可以进一步修改其属性，其中许多属性是使用container.getContainerProperties（）设置的。以下示例配置ConcurrentMessageListenerContainer：
@Bean
公共ConcurrentMessageListenerContainer(
ConcurrentKafkalistener集装箱工厂）{
ConcurrentMessageListenerContainer容器=
factory.createContainer（“topic1”、“topic2”）；
setMessageListener（m->{…}）；
返回容器；
}
注意：以这种方式创建的容器不会添加到端点注册表中。它们应该创建为@Bean定义，以便在应用程序上下文中注册。您可以使用自定义分区器来分配您想要的分区。这是卡夫卡的消费品 编辑 看 它适用于
@JmsListener@KafkaListenerInstanceAlreadyExistsException@SpringBootApplication
@EnableScheduling
@Slf4j
public class KafkaConsumerApp {
public static void main(String[] args) {
log.info("Starting spring boot KafkaConsumerApp..");
SpringApplication.run(KafkaConsumerApp.class, args);
}
}
@EnableKafka
@Configuration
public class KafkaConfiguration {
private final KafkaProperties kafkaProperties;
@Value("${kafka.brokers:localhost:9092}")
private String bootstrapServer;
@Value("${kafka.consumerClientId}")
private String consumerClientId;
@Value("${kafka.consumerGroupId}")
private String consumerGroupId;
@Value("${kafka.topicMonitorClientId}")
private String topicMonitorClientId;
@Value("${kafka.topicMonitorGroupId}")
private String topicMonitorGroupId;
@Autowired
private ConfigurableApplicationContext context;
@Autowired
public KafkaConfiguration( KafkaProperties kafkaProperties ) {
this.kafkaProperties = kafkaProperties;
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory( consumerFactory( consumerClientId, consumerGroupId ) );
factory.getContainerProperties().setAckMode( ContainerProperties.AckMode.MANUAL );
return factory;
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> topicMonitorContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory( consumerFactory( topicMonitorClientId, topicMonitorGroupId ) );
factory.getContainerProperties().setAckMode( ContainerProperties.AckMode.MANUAL );
factory.getContainerProperties().setConsumerRebalanceListener( new KafkaRebalanceListener( context ) );
return factory;
}
private ConsumerFactory<String, String> consumerFactory( String clientId, String groupId ) {
Map<String, Object> config = new HashMap<>();
config.putAll( kafkaProperties.buildConsumerProperties() );
config.put( ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer );
config.put( ConsumerConfig.CLIENT_ID_CONFIG, clientId );
config.put( ConsumerConfig.GROUP_ID_CONFIG, groupId );
config.put( ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false ); // needs to be turned off for rebalancing during topic addition and deletion
// check -> https://stackoverflow.com/questions/56264681/is-it-possible-to-have-one-kafka-consumer-thread-per-topic/56274988?noredirect=1#comment99401765_56274988
return new DefaultKafkaConsumerFactory<>( config, new StringDeserializer(), new StringDeserializer() );
}
}
@Configuration
public class KafkaListenerConfiguration {
@Bean
@Scope("prototype")
public KafkaMessageListener kafkaMessageListener() {
return new KafkaMessageListener();
}
}
@Slf4j
public class KafkaMessageListener {
/*
* This is the actual message listener that will process messages. It will be instantiated per topic.
*/
@KafkaListener( topics = "${topic}", containerFactory = "kafkaListenerContainerFactory" )
public void receiveHyperscalerMessage( ConsumerRecord<String, String> record, Acknowledgment acknowledgment, Consumer<String, String> consumer ) {
log.debug("Kafka message - ThreadName={}, Hashcode={}, Partition={}, Topic={}, Value={}",
Thread.currentThread().getName(), Thread.currentThread().hashCode(), record.partition(), record.topic(), record.value() );
// do processing
// this is just a sample acknowledgment. it can be optimized to acknowledge after processing a batch of messages.
acknowledgment.acknowledge();
}
}
@Service
public class KafkaTopicMonitor {
/*
* The main purpose of this listener is to detect the rebalance events on our topic pattern, so that
* we can create a listener bean instance (consumer thread) per topic.
*
* Note that we use the wildcard topic pattern here.
*/
@KafkaListener( topicPattern = ".*abc.def.ghi", containerFactory = "topicMonitorContainerFactory" )
public void monitorTopics( ConsumerRecord<String, String> record ) {
// do nothing
}
}
@Slf4j
public class KafkaRebalanceListener implements ConsumerAwareRebalanceListener {
private static final ConcurrentMap<String, KafkaMessageListener> listenerMap = new ConcurrentHashMap<>();
private final ConfigurableApplicationContext context;
public KafkaRebalanceListener( ConfigurableApplicationContext context ) {
this.context = context;
}
public void onPartitionsRevokedBeforeCommit(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
// do nothing
}
public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
// do nothing
}
public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
log.info("OnPartitionsAssigned - partitions={} - {}", partitions.size(), partitions);
Properties props = new Properties();
context.getEnvironment().getPropertySources().addLast( new PropertiesPropertySource("topics", props) );
for( TopicPartition tp: partitions ) {
listenerMap.computeIfAbsent( tp.topic(), key -> {
log.info("Creating messageListener bean instance for topic - {}", key );
props.put( "topic", key );
// create new KafkaMessageListener bean instance
return context.getBean( "kafkaMessageListener", KafkaMessageListener.class );
});
}
}
}
@springboot应用程序
@使能调度
@Slf4j
公共类卡夫卡消费品{
公共静态void main（字符串[]args）{
log.info（“Starting spring boot KafkaConsumerApp..”；
run（KafkaConsumerApp.class，args）；
}
}
@使能卡夫卡
@配置
公共类卡夫卡配置{
私人最终卡夫卡财产卡夫卡财产；
@值（${kafka.brokers:localhost:9092}”）
私有字符串引导服务器；
@值（${kafka.consumerClientId}”）
私有字符串consumerClientId；
@值（${kafka.consumerGroupId}”）
私有字符串consumerGroupId；
@值（${kafka.topicMonitorClientId}”）
私有字符串topicMonitorClientId；
@值（${kafka.topicMonitorGroupId}”）
私有字符串topicMonitorGroupId；
@自动连线
私有配置应用程序上下文上下文；
@自动连线
公共卡夫卡配置（卡夫卡财产卡夫卡财产）{
this.kafkaProperties=kafkaProperties；
}
@豆子
公共ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory（）{
ConcurrentKafkListenerContainerFactory=新ConcurrentKafkListenerContainerFactory（）；
setConsumerFactory（consumerFactory（consumerClientId，consumerGroupId））；
factory.getContainerProperties（）.setAckMode（ContainerProperties.AckMode.MANUAL）；
返回工厂；
}
@豆子
公共ConcurrentKafkaListenerContainerFactory topicMonitorContainerFactory（）{
ConcurrentKafkListenerContainerFactory=新ConcurrentKafkListenerContainerFactory（）；
setConsumerFactory（consumerFactory（topicMonitorClientId，topicMonitorGroupId））；
factory.getContainerProperties（）.setAckMode（ContainerProperties.AckMode.MANUAL）；
factory.getContainerProperties（）.setConsumerBalanceListener（新的KafkaRebalanceListener（上下文））；
返回工厂；
}
私有ConsumerFactory ConsumerFactory（字符串clientId、字符串groupId）{
Map config=newhashmap（）；
config.putAll（kafkaProperties.buildConsumerProperties（））；
config.put（ConsumerConfig.BOOTSTRAP\u SERVERS\u config，bootstrapServer）；
config.put（ConsumerConfig.CLIENT\u ID\u config，clientId）；
config.put（ConsumerConfig.GROUP\u ID\u config，groupId）；
config.put（ConsumerConfig.ENABLE_AUTO_COMMIT_config，false）；//在添加和删除主题期间需要关闭以重新平衡
//检查->https://stackoverflow.com/questions/56264681/is-it-possible-to-have-one-kafka-consume
@SpringBootApplication
@EnableScheduling
@Slf4j
public class KafkaConsumerApp {
public static void main(String[] args) {
log.info("Starting spring boot KafkaConsumerApp..");
SpringApplication.run(KafkaConsumerApp.class, args);
}
}
@EnableKafka
@Configuration
public class KafkaConfiguration {
private final KafkaProperties kafkaProperties;
@Value("${kafka.brokers:localhost:9092}")
private String bootstrapServer;
@Value("${kafka.consumerClientId}")
private String consumerClientId;
@Value("${kafka.consumerGroupId}")
private String consumerGroupId;
@Value("${kafka.topicMonitorClientId}")
private String topicMonitorClientId;
@Value("${kafka.topicMonitorGroupId}")
private String topicMonitorGroupId;
@Autowired
private ConfigurableApplicationContext context;
@Autowired
public KafkaConfiguration( KafkaProperties kafkaProperties ) {
this.kafkaProperties = kafkaProperties;
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory( consumerFactory( consumerClientId, consumerGroupId ) );
factory.getContainerProperties().setAckMode( ContainerProperties.AckMode.MANUAL );
return factory;
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> topicMonitorContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory( consumerFactory( topicMonitorClientId, topicMonitorGroupId ) );
factory.getContainerProperties().setAckMode( ContainerProperties.AckMode.MANUAL );
factory.getContainerProperties().setConsumerRebalanceListener( new KafkaRebalanceListener( context ) );
return factory;
}
private ConsumerFactory<String, String> consumerFactory( String clientId, String groupId ) {
Map<String, Object> config = new HashMap<>();
config.putAll( kafkaProperties.buildConsumerProperties() );
config.put( ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer );
config.put( ConsumerConfig.CLIENT_ID_CONFIG, clientId );
config.put( ConsumerConfig.GROUP_ID_CONFIG, groupId );
config.put( ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false ); // needs to be turned off for rebalancing during topic addition and deletion
// check -> https://stackoverflow.com/questions/56264681/is-it-possible-to-have-one-kafka-consumer-thread-per-topic/56274988?noredirect=1#comment99401765_56274988
return new DefaultKafkaConsumerFactory<>( config, new StringDeserializer(), new StringDeserializer() );
}
}
@Configuration
public class KafkaListenerConfiguration {
@Bean
@Scope("prototype")
public KafkaMessageListener kafkaMessageListener() {
return new KafkaMessageListener();
}
}
@Slf4j
public class KafkaMessageListener {
/*
* This is the actual message listener that will process messages. It will be instantiated per topic.
*/
@KafkaListener( topics = "${topic}", containerFactory = "kafkaListenerContainerFactory" )
public void receiveHyperscalerMessage( ConsumerRecord<String, String> record, Acknowledgment acknowledgment, Consumer<String, String> consumer ) {
log.debug("Kafka message - ThreadName={}, Hashcode={}, Partition={}, Topic={}, Value={}",
Thread.currentThread().getName(), Thread.currentThread().hashCode(), record.partition(), record.topic(), record.value() );
// do processing
// this is just a sample acknowledgment. it can be optimized to acknowledge after processing a batch of messages.
acknowledgment.acknowledge();
}
}
@Service
public class KafkaTopicMonitor {
/*
* The main purpose of this listener is to detect the rebalance events on our topic pattern, so that
* we can create a listener bean instance (consumer thread) per topic.
*
* Note that we use the wildcard topic pattern here.
*/
@KafkaListener( topicPattern = ".*abc.def.ghi", containerFactory = "topicMonitorContainerFactory" )
public void monitorTopics( ConsumerRecord<String, String> record ) {
// do nothing
}
}
@Slf4j
public class KafkaRebalanceListener implements ConsumerAwareRebalanceListener {
private static final ConcurrentMap<String, KafkaMessageListener> listenerMap = new ConcurrentHashMap<>();
private final ConfigurableApplicationContext context;
public KafkaRebalanceListener( ConfigurableApplicationContext context ) {
this.context = context;
}
public void onPartitionsRevokedBeforeCommit(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
// do nothing
}
public void onPartitionsRevokedAfterCommit(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
// do nothing
}
public void onPartitionsAssigned(Consumer<?, ?> consumer, Collection<TopicPartition> partitions) {
log.info("OnPartitionsAssigned - partitions={} - {}", partitions.size(), partitions);
Properties props = new Properties();
context.getEnvironment().getPropertySources().addLast( new PropertiesPropertySource("topics", props) );
for( TopicPartition tp: partitions ) {
listenerMap.computeIfAbsent( tp.topic(), key -> {
log.info("Creating messageListener bean instance for topic - {}", key );
props.put( "topic", key );
// create new KafkaMessageListener bean instance
return context.getBean( "kafkaMessageListener", KafkaMessageListener.class );
});
}
}
}