Apache spark 由于bigendian on spark测试导致的故障
下面是在spark上失败的测试套件。它看起来像是特定于big endian的，因为它正在传递给little endian系统。在这些方面的任何投入都将非常有用 testcase源代码：Apache spark 由于bigendian on spark测试导致的故障,apache-spark,apache-spark-sql,spark-streaming,Apache Spark,Apache Spark Sql,Spark Streaming,下面是在spark上失败的测试套件。它看起来像是特定于big endian的，因为它正在传递给little endian系统。在这些方面的任何投入都将非常有用 testcase源代码： test("flatMapGroupsWithState - recovery from checkpoint uses state format version 1") { // Function to maintain the max event time as state and set the t
test("flatMapGroupsWithState - recovery from checkpoint uses state format version 1") {
// Function to maintain the max event time as state and set the timeout timestamp based on the
// current max event time seen. It returns the max event time in the state, or -1 if the state
// was removed by timeout.
val stateFunc = (key: String, values: Iterator[(String, Long)], state: GroupState[Long]) => {
assertCanGetProcessingTime { state.getCurrentProcessingTimeMs() >= 0 }
assertCanGetWatermark { state.getCurrentWatermarkMs() >= -1 }
val timeoutDelaySec = 5
if (state.hasTimedOut) {
state.remove()
Iterator((key, -1))
} else {
val valuesSeq = values.toSeq
val maxEventTimeSec = math.max(valuesSeq.map(_._2).max, state.getOption.getOrElse(0L))
val timeoutTimestampSec = maxEventTimeSec + timeoutDelaySec
state.update(maxEventTimeSec)
state.setTimeoutTimestamp(timeoutTimestampSec * 1000)
Iterator((key, maxEventTimeSec.toInt))
}
}
val inputData = MemoryStream[(String, Int)]
val result =
inputData.toDS
.select($"_1".as("key"), $"_2".cast("timestamp").as("eventTime"))
.withWatermark("eventTime", "10 seconds")
.as[(String, Long)]
.groupByKey(_._1)
.flatMapGroupsWithState(Update, EventTimeTimeout)(stateFunc)
val resourceUri = this.getClass.getResource(
"/structured-streaming/checkpoint-version-2.3.1-flatMapGroupsWithState-state-format-1/").toURI
val checkpointDir = Utils.createTempDir().getCanonicalFile
// Copy the checkpoint to a temp dir to prevent changes to the original.
// Not doing this will lead to the test passing on the first run, but fail subsequent runs.
FileUtils.copyDirectory(new File(resourceUri), checkpointDir)
inputData.addData(("a", 11), ("a", 13), ("a", 15))
inputData.addData(("a", 4))
testStream(result, Update)(
StartStream(
checkpointLocation = checkpointDir.getAbsolutePath,
additionalConfs = Map(SQLConf.FLATMAPGROUPSWITHSTATE_STATE_FORMAT_VERSION.key -> "2")),
/*
Note: The checkpoint was generated using the following input in Spark version 2.3.1
AddData(inputData, ("a", 11), ("a", 13), ("a", 15)),
// Max event time = 15. Timeout timestamp for "a" = 15 + 5 = 20. Watermark = 15 - 10 = 5.
CheckNewAnswer(("a", 15)), // Output = max event time of a
AddData(inputData, ("a", 4)), // Add data older than watermark for "a"
CheckNewAnswer(), // No output as data should get filtered by watermark
*/
AddData(inputData, ("a", 10)), // Add data newer than watermark for "a"
CheckNewAnswer(("a", 15)), // Max event time is still the same
// Timeout timestamp for "a" is still 20 as max event time for "a" is still 15.
// Watermark is still 5 as max event time for all data is still 15.
Execute { query =>
// Verify state format = 1
val f = query.lastExecution.executedPlan.collect { case f: FlatMapGroupsWithStateExec => f }
assert(f.size == 1)
assert(f.head.stateFormatVersion == 1)
},
AddData(inputData, ("b", 31)), // Add data newer than watermark for "b", not "a"
// Watermark = 31 - 10 = 21, so "a" should be timed out as timeout timestamp for "a" is 20.
CheckNewAnswer(("a", -1), ("b", 31)) // State for "a" should timeout and emit -1
)
}
[info] == Results ==
[info] !== Correct Answer - 1 == == Spark Answer - 1 ==
[info] struct<_1:string,_2:int> struct<_1:string,_2:int>
[info] ![a,15] [a,10]
[info]
[info]
[info] == Progress ==
[info] StartStream(ProcessingTime(0),org.apache.spark.util.SystemClock@e140d1ac,Map(spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion -> 2),/home/test/spark/target/tmp/spark-16dab6bf-9077-4bca-bbf4-4a3083438a5d)
[info] AddData to MemoryStream[_1#2303,_2#2304]: (a,10)
[info] => CheckNewAnswer: [a,15]
[info] AssertOnQuery(<condition>, name)
[info] AddData to MemoryStream[_1#2303,_2#2304]: (b,31)
[info] CheckNewAnswer: [a,-1],[b,31]
[info]
[info] == Stream ==
[info] Output Mode: Update
[info] Stream state: {MemoryStream[_1#2303,_2#2304]: 2}
[info] Thread state: alive
[info] Thread stack trace: java.lang.Thread.sleep(Native Method)
[info] java.lang.Thread.sleep(Thread.java:943)
[info] org.apache.spark.sql.execution.streaming.MicroBatchExecution$$anonfun$runActivatedStream$1.apply$mcZ$sp(MicroBatchExecution.scala:215)
[info] org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute(TriggerExecutor.scala:56)
[info] org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream(MicroBatchExecution.scala:160)
[info] org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:281)
[info] org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:193)
[info]
[info]
[info] == Sink ==
[info] 2: [a,10]
[info]
[info]
[info] == Plan ==
[info] == Parsed Logical Plan ==
[info] SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1, true, false) AS _1#2319, assertnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2 AS _2#2320]
[info] +- FlatMapGroupsWithState <function3>, cast(value#2314 as string).toString, newInstance(class scala.Tuple2), [value#2314], [key#2306, eventTime#2307-T10000ms], obj#2318: scala.Tuple2, class[value[0]: bigint], Update, false, EventTimeTimeout
[info] +- AppendColumns <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,LongType,false)], newInstance(class scala.Tuple2), [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#2314]
[info] +- EventTimeWatermark eventTime#2307: timestamp, interval 10 seconds
[info] +- Project [_1#2303 AS key#2306, cast(_2#2304 as timestamp) AS eventTime#2307]
[info] +- Project [_1#2338 AS _1#2303, _2#2339 AS _2#2304]
[info] +- Streaming RelationV2 MemoryStreamDataSource$[_1#2338, _2#2339]
[info]
[info] == Analyzed Logical Plan ==
[info] _1: string, _2: int
[info] SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(assertnotnull(input[0, scala.Tuple2, true]))._1, true, false) AS _1#2319, assertnotnull(assertnotnull(input[0, scala.Tuple2, true]))._2 AS _2#2320]
[info] +- FlatMapGroupsWithState <function3>, cast(value#2314 as string).toString, newInstance(class scala.Tuple2), [value#2314], [key#2306, eventTime#2307-T10000ms], obj#2318: scala.Tuple2, class[value[0]: bigint], Update, false, EventTimeTimeout
[info] +- AppendColumns <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,LongType,false)], newInstance(class scala.Tuple2), [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#2314]
[info] +- EventTimeWatermark eventTime#2307: timestamp, interval 10 seconds
[info] +- Project [_1#2303 AS key#2306, cast(_2#2304 as timestamp) AS eventTime#2307]
[info] +- Project [_1#2338 AS _1#2303, _2#2339 AS _2#2304]
[info] +- Streaming RelationV2 MemoryStreamDataSource$[_1#2338, _2#2339]
[info]
[info] == Optimized Logical Plan ==
[info] SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, scala.Tuple2, true])._1, true, false) AS _1#2319, assertnotnull(input[0, scala.Tuple2, true])._2 AS _2#2320]
[info] +- FlatMapGroupsWithState <function3>, value#2314.toString, newInstance(class scala.Tuple2), [value#2314], [key#2306, eventTime#2307-T10000ms], obj#2318: scala.Tuple2, class[value[0]: bigint], Update, false, EventTimeTimeout
[info] +- AppendColumns <function1>, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,LongType,false)], newInstance(class scala.Tuple2), [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#2314]
[info] +- EventTimeWatermark eventTime#2307: timestamp, interval 10 seconds
[info] +- Project [_1#2338 AS key#2306, cast(_2#2339 as timestamp) AS eventTime#2307]
[info] +- Streaming RelationV2 MemoryStreamDataSource$[_1#2338, _2#2339]
[info]
[info] == Physical Plan ==
[info] *(3) SerializeFromObject [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, assertnotnull(input[0, scala.Tuple2, true])._1, true, false) AS _1#2319, assertnotnull(input[0, scala.Tuple2, true])._2 AS _2#2320]
[info] +- FlatMapGroupsWithState <function3>, value#2314.toString, newInstance(class scala.Tuple2), [value#2314], [key#2306, eventTime#2307-T10000ms], obj#2318: scala.Tuple2, state info [ checkpoint = file:/home/test/spark/target/tmp/spark-16dab6bf-9077-4bca-bbf4-4a3083438a5d/state, runId = 9cc5efc0-97c5-48e6-89d5-66330069026a, opId = 0, ver = 2, numPartitions = 5], class[value[0]: bigint], 1, Update, EventTimeTimeout, 1575008112198, 5000
[info] +- *(2) Sort [value#2314 ASC NULLS FIRST], false, 0
[info] +- Exchange hashpartitioning(value#2314, 5)
[info] +- AppendColumns <function1>, newInstance(class scala.Tuple2), [staticinvoke(class org.apache.spark.unsafe.types.UTF8String, StringType, fromString, input[0, java.lang.String, true], true, false) AS value#2314]
[info] +- EventTimeWatermark eventTime#2307: timestamp, interval 10 seconds
[info] +- *(1) Project [_1#2338 AS key#2306, cast(_2#2339 as timestamp) AS eventTime#2307]
[info] +- *(1) Project [_1#2338, _2#2339]
[info] +- *(1) ScanV2 MemoryStreamDataSource$[_1#2338, _2#2339] (StreamTest.scala:450)
[info] org.scalatest.exceptions.TestFailedException:
[info] at org.scalatest.Assertions$class.newAssertionFailedException(Assertions.scala:528)
[info] at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
[info] at org.scalatest.Assertions$class.fail(Assertions.scala:1089)
[info] at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
[info] at org.apache.spark.sql.streaming.StreamTest$class.failTest$1(StreamTest.scala:450)
[info] at org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$20.apply(StreamTest.scala:751)
[info] at org.apache.spark.sql.streaming.StreamTest$$anonfun$executeAction$1$20.apply(StreamTest.scala:751)
[info] at scala.Option.foreach(Option.scala:257)
[info] at org.apache.spark.sql.streaming.StreamTest$class.executeAction$1(StreamTest.scala:750)
[info] at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:770)
[info] at org.apache.spark.sql.streaming.StreamTest$$anonfun$liftedTree1$1$1.apply(StreamTest.scala:757)
[info] at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
[info] at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35)
[info] at org.apache.spark.sql.streaming.StreamTest$class.liftedTree1$1(StreamTest.scala:757)
[info] at org.apache.spark.sql.streaming.StreamTest$class.testStream(StreamTest.scala:756)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite.testStream(FlatMapGroupsWithStateSuite.scala:48)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite$$anonfun$21.apply$mcV$sp(FlatMapGroupsWithStateSuite.scala:924)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite$$anonfun$21.apply(FlatMapGroupsWithStateSuite.scala:883)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite$$anonfun$21.apply(FlatMapGroupsWithStateSuite.scala:883)
[info] at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
[info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
[info] at org.scalatest.Transformer.apply(Transformer.scala:22)
[info] at org.scalatest.Transformer.apply(Transformer.scala:20)
[info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
[info] at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:103)
[info] at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:183)
[info] at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
[info] at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
[info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
[info] at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:196)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(FlatMapGroupsWithStateSuite.scala:48)
[info] at org.scalatest.BeforeAndAfterEach$class.runTest(BeforeAndAfterEach.scala:221)
[info] at org.apache.spark.sql.streaming.FlatMapGroupsWithStateSuite.runTest(FlatMapGroupsWithStateSuite.scala:48)
[info] at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
[info] at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
[info] at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:396)
[info] at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384)
[info] at scala.collection.immutable.List.foreach(List.scala:392)
[info] at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
[info] at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:379)
[info] at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
[info] at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:229)
[info] at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
[info] at org.scalatest.Suite$class.run(Suite.scala:1147)
[info] at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
[info] at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
[info] at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
[info] at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
[info] at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:233)
[info] at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:52)
[info] at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213)
[info] at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210)
[info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:52)
[info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314)
[info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:480)
[info] at sbt.ForkMain$Run$2.call(ForkMain.java:296)
[info] at sbt.ForkMain$Run$2.call(ForkMain.java:286)
[info] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[info] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[info] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[info] at java.lang.Thread.run(Thread.java:813)
[info]==结果==
[信息]！==正确答案-1===火花答案-1==
[信息]结构
[信息]！[甲,十五][甲,十]
[信息]
[信息]
[信息]==进度==
[信息]StartStream（ProcessingTime（0），org.apache.spark.util。SystemClock@e140d1ac，映射（spark.sql.streaming.flatMapGroupsWithState.stateFormatVersion->2），/home/test/spark/target/tmp/spark-16dab6bf-9077-4bca-bbf4-4a3083438a5d）
[信息]将数据添加到MemoryStream[_1#2303，_2#2304]：（a，10）
[info]=>CheckNewAnswer:[a，15]
[信息]资产查询（，名称）
[信息]将数据添加到MemoryStream[_1#2303，_2#2304]：（b，31）
[信息]检查新答案：[a，-1]，[b，31]
[信息]
[信息]==流==
[信息]输出模式：更新
[信息]流状态：{MemoryStream[#1#2303，#2#2304]：2}
[信息]线程状态：活动
[信息]线程堆栈跟踪：java.lang.Thread.sleep（本机方法）
[信息]java.lang.Thread.sleep（Thread.java:943）
[信息]org.apache.spark.sql.execution.streaming.MicroBatchExecution$$anonfun$runActivatedStream$1.apply$mcZ$sp（MicroBatchExecution.scala:215）
[info]org.apache.spark.sql.execution.streaming.ProcessingTimeExecutor.execute（TriggerExecutor.scala:56）
[信息]org.apache.spark.sql.execution.streaming.MicroBatchExecution.runActivatedStream（MicroBatchExecution.scala:160）
[信息]org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$streaming$StreamExecution$$runStream（StreamExecution.scala:281）
[info]org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run（StreamExecution.scala:193）
[信息]
[信息]
[信息]==接收器==
[信息]2:[a，10]
[信息]
[信息]
[信息]==计划==
[信息]==已解析的逻辑计划==
[info]SerializeFromObject[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，assertnotnull（assertnotnull（输入[0，scala.Tuple2，true]））。_1，true，false）为2319，assertnotnull（assertnotnull（输入[0，scala.Tuple2，true]））。_2为2320]
[info]+-FlatMapGroupsWithState，cast（值2314作为字符串）。toString，newInstance（类scala.Tuple2），[value#2314]，[key#2306，eventTime#2307-T10000ms]，obj#2318:scala.Tuple2，类[value[0]：bigint]，Update，false，EventTimeTimeout
[info]+-AppendColumns，类scala.Tuple2，[StructField（_1，StringType，true），StructField（_2，LongType，false）]，newInstance（类scala.Tuple2），[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，输入[0，java.lang.String，true]，true，false）作为值#2314]
[info]+-EventTimeWatermark eventTime#2307：时间戳，间隔10秒
[info]+-Project[_1#2303作为键#2306，cast（_2#2304作为时间戳）作为事件时间#2307]
[信息]+-项目[_1#2338 AS#u 1#2303，_2#2339 AS#u 2#2304]
[info]+-streamingrelationv2memoryStreamDataSource$[[u1#2338，[u2#2339]
[信息]
[信息]==分析的逻辑计划==
[信息]_1:string，_2:int
[info]SerializeFromObject[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，assertnotnull（assertnotnull（输入[0，scala.Tuple2，true]））。_1，true，false）为2319，assertnotnull（assertnotnull（输入[0，scala.Tuple2，true]））。_2为2320]
[info]+-FlatMapGroupsWithState，cast（值2314作为字符串）。toString，newInstance（类scala.Tuple2），[value#2314]，[key#2306，eventTime#2307-T10000ms]，obj#2318:scala.Tuple2，类[value[0]：bigint]，Update，false，EventTimeTimeout
[info]+-AppendColumns，类scala.Tuple2，[StructField（_1，StringType，true），StructField（_2，LongType，false）]，newInstance（类scala.Tuple2），[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，输入[0，java.lang.String，true]，true，false）作为值#2314]
[info]+-EventTimeWatermark eventTime#2307：时间戳，间隔10秒
[info]+-Project[_1#2303作为键#2306，cast（_2#2304作为时间戳）作为事件时间#2307]
[信息]+-项目[_1#2338 AS#u 1#2303，_2#2339 AS#u 2#2304]
[info]+-streamingrelationv2memoryStreamDataSource$[[u1#2338，[u2#2339]
[信息]
[信息]==优化的逻辑计划==
[info]序列化FromObject[staticinvoke（类org.apache.spark.safe.types.UTF8String，StringType，fromString，assertnotnull（输入[0，scala.Tuple2，true]）。\u 1，true，false）为2319，assertnotnull（输入[0，scala.Tuple2，true]）。\u 2为2320]
[info]+-FlatMapGroupsWithState，value#2314.toString，newInstance（类scala.Tuple2），[value#2314]，[key#2306，eventTime#2307-T10000ms]，obj#2318:scala.Tuple2，类[value[0]：bigint]，Update，false，EventTimeTimeout
[info]+-AppendColumns，类scala.Tuple2，[StructField（_1，StringType，true），StructField（_2，LongType，false）]，newInstance（类scala.Tuple2），[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，输入[0，java.lang.String，true]，true，false）作为值#2314]
[info]+-EventTimeWatermark eventTime#2307：时间戳，间隔10秒
[info]+-Project[_1#2338作为键#2306，cast（_2#2339作为时间戳）作为事件时间#2307]
[info]+-streamingrelationv2memoryStreamDataSource$[[u1#2338，[u2#2339]
[信息]
[信息]==实际计划==
[info]*（3）SerializeFromObject[staticinvoke（类org.apache.spark.unsafe.types.UTF8String，StringType，fromString，assertnotnull（输入[0，scala.Tuple2，true]）。_1，true，false）作为#2319，assertnotnull（输入[0，scala.Tuple2，true]）。_2作为#2320]
[info]+-FlatMapGroupsWithState，value#2314.toString，newInstance（类scala.Tuple2），[value#2314]，[key#2306，eventTime#2307-T10000ms]，obj#2318:scala.Tuple2，状态信息[checkpoint=file:/home/test/spark/target/tmp/spark-16dab6bf-9077-4bca-bbf4-4a30838d/state，