Apache spark Spark Streaming-从检查点重新启动
我们正在构建一个容错系统，该系统可以读取卡夫卡，并编写HBase和HDFS。批次每5秒运行一次。下面是我们希望设置的场景：Apache spark Spark Streaming-从检查点重新启动,apache-spark,spark-streaming,Apache Spark,Spark Streaming,我们正在构建一个容错系统，该系统可以读取卡夫卡，并编写HBase和HDFS。批次每5秒运行一次。下面是我们希望设置的场景： 在启用检查点的情况下启动新的spark流式处理，从kafka读取数据，处理数据并存储到HDFS和HBase 杀死spark流媒体作业，消息继续流入Kafka 重新启动spark streaming作业，下面是我们真正希望发生的事情：spark streaming读取检查点数据并使用正确的kafka偏移量重新启动。即使spark streaming作业已终止并重新启动，也不会
val ddqSsc = StreamingContext.getOrCreate(checkpointDir, () =>
createDDQStreamingContext(slideInterval.toLong, inputKafka, outputKafka, hbaseVerTableName, checkpointDir, baseRawHdfs, securityProtocol, groupID, zooKeeper, kafkaBrokers, hiveDBToLoad, hiveTableToLoad))
def createDDQStreamingContext(slideInterval: Long, inputKafka: String, outputKafka: String, hbaseVerTableName: String, checkpointDir: String, baseRawHdfs: String, securityProtocol: String, groupID: String, zooKeeper: String, kafkaBrokers: String, hiveDBToLoad: String, hiveTableToLoad: String): StreamingContext = {
val sparkConf = new SparkConf()
val ssc = new StreamingContext(sparkConf, Seconds(slideInterval))
//val sqlContext = new SQLContext(sc)
val sqlContext = new HiveContext(ssc.sparkContext)
import sqlContext.implicits._
ssc.checkpoint(checkpointDir)
val kafkaTopics = Set(inputKafka)
//Kafka parameters
var kafkaParams = Map[String, String]()
kafkaParams += ("bootstrap.servers" -> kafkaBrokers)
kafkaParams += ("zookeeper.connect" -> zooKeeper)
//Need this in a kerberos environment
kafkaParams += ("security.protocol" -> securityProtocol)
kafkaParams += ("sasl.kerberos.service.name" -> "kafka")
//WHAT IS THIS!!??
kafkaParams += ("group.id" -> groupID)
kafkaParams += ("key.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer")
kafkaParams += ("value.deserializer" -> "org.apache.kafka.common.serialization.StringDeserializer")
val inputDataDstream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, kafkaTopics)