Scala 如何使用Spark Structure Streaming的writeStream()方法获取本地系统中生成的.csv文件列表
我想知道如何从我的程序中获取.csv文件列表Scala 如何使用Spark Structure Streaming的writeStream()方法获取本地系统中生成的.csv文件列表,scala,apache-spark,Scala,Apache Spark,我想知道如何从我的程序中获取.csv文件列表 class LiveStreamingProg { def liveData(sparkSession: SparkSession, sparkCont: SparkContext): Unit = { val ss = new StreamingContext(sparkCont, Seconds(2)) val path = "s3a://word.count.spark
class LiveStreamingProg {
def liveData(sparkSession: SparkSession, sparkCont: SparkContext): Unit = {
val ss = new StreamingContext(sparkCont, Seconds(2))
val path = "s3a://word.count.spark/";
val schema = StructType(
StructField("date", StringType, true) ::
StructField("close",StringType, true) ::
StructField("volume", StringType, true) ::
StructField("open", StringType, true) ::
StructField("high",StringType,true) ::
StructField("low", StringType,true) :: Nil)
val line=sparkSession.readStream
.format("csv")
.option("sep",",").schema(schema)
.load(path)
val l1=line.writeStream.outputMode("append")
.format("csv")
.option("checkpointLocation", "/home/user/StockDataSparkStraming/src/main/scala/CheckPoint")
.trigger(Trigger.ProcessingTime("5 seconds"))
.start("/home/user/StockDataSparkStraming/src/main/scala/output")
l1.awaitTermination()
}
}
我无法调用readCSV()方法,因为我需要所有.csv文件列表