Scala 如何访问JSON文件中的子实体？_Scala_Apache Spark_Apache Spark Sql

Scala 如何访问JSON文件中的子实体？

scala apache-spark

Scala 如何访问JSON文件中的子实体？,scala,apache-spark,apache-spark-sql,Scala,Apache Spark,Apache Spark Sql,我有一个json文件，如下所示： { "employeeDetails":{ "name": "xxxx", "num":"415" }, "work":[ { "monthYear":"01/2007", "workdate":"1|2|3|....|31", "workhours":"8|8|8....|8" }, { "monthYear":"02/2007", "workdat

我有一个json文件，如下所示：

{
  "employeeDetails":{
    "name": "xxxx",
    "num":"415"
  },
  "work":[
    {
      "monthYear":"01/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    },
    {
      "monthYear":"02/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    }
  ]
}

import org.apache.spark.{SparkConf, SparkContext}

object JSON2 {
  def main (args: Array[String]) {
    val spark =
      SparkSession.builder()
        .appName("SQL-JSON")
        .master("local[4]")
        .getOrCreate()

    import spark.implicits._

    val employees = spark.read.json("sample.json")
    employees.printSchema()
    employees.select("employeeDetails").show()
  }
}

Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve '`employeeDetails`' given input columns: [_corrupt_record];;
'Project ['employeeDetails]
+- Relation[_corrupt_record#0] json

我必须从这个json数据中获取工作日期和工作时间

我试着这样做：

{
  "employeeDetails":{
    "name": "xxxx",
    "num":"415"
  },
  "work":[
    {
      "monthYear":"01/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    },
    {
      "monthYear":"02/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    }
  ]
}

import org.apache.spark.{SparkConf, SparkContext}

object JSON2 {
  def main (args: Array[String]) {
    val spark =
      SparkSession.builder()
        .appName("SQL-JSON")
        .master("local[4]")
        .getOrCreate()

    import spark.implicits._

    val employees = spark.read.json("sample.json")
    employees.printSchema()
    employees.select("employeeDetails").show()
  }
}

Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve '`employeeDetails`' given input columns: [_corrupt_record];;
'Project ['employeeDetails]
+- Relation[_corrupt_record#0] json

我有这样的例外：

{
  "employeeDetails":{
    "name": "xxxx",
    "num":"415"
  },
  "work":[
    {
      "monthYear":"01/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    },
    {
      "monthYear":"02/2007",
      "workdate":"1|2|3|....|31",
      "workhours":"8|8|8....|8"
    }
  ]
}

import org.apache.spark.{SparkConf, SparkContext}

object JSON2 {
  def main (args: Array[String]) {
    val spark =
      SparkSession.builder()
        .appName("SQL-JSON")
        .master("local[4]")
        .getOrCreate()

    import spark.implicits._

    val employees = spark.read.json("sample.json")
    employees.printSchema()
    employees.select("employeeDetails").show()
  }
}

Exception in thread "main" org.apache.spark.sql.AnalysisException: cannot resolve '`employeeDetails`' given input columns: [_corrupt_record];;
'Project ['employeeDetails]
+- Relation[_corrupt_record#0] json

我是新手

给定的输入列：[[u corrupt_record]

原因是Spark支持JSON文件，其中“每行必须包含一个单独的、自包含的有效JSON对象。”

引述：

请注意，作为json文件提供的文件不是典型的json文件。每行必须包含一个单独的、自包含的有效JSON对象。有关更多信息，请参阅JSON行文本格式，也称为换行分隔JSON。因此，常规的多行JSON文件通常会失败

如果Spark的JSON文件不正确，它将存储在

\u corrupt\u record

下（您可以使用

columnNameOfCorruptRecord

选项更改）

您的文件是不正确的，不仅因为它是一个多行JSON，而且因为（一个轻量级且灵活的命令行JSON处理器）这样说

修复JSON文件后，使用以下技巧加载多行JSON文件

scala> spark.version
res5: String = 2.1.1

val employees = spark.read.json(sc.wholeTextFiles("employee.json").values)
scala> employees.printSchema
root
 |-- employeeDetails: struct (nullable = true)
 |    |-- name: string (nullable = true)
 |    |-- num: string (nullable = true)
 |-- work: array (nullable = true)
 |    |-- element: struct (containsNull = true)
 |    |    |-- monthYear: string (nullable = true)
 |    |    |-- workdate: string (nullable = true)
 |    |    |-- workhours: string (nullable = true)

scala> employees.select("employeeDetails").show()
+---------------+
|employeeDetails|
+---------------+
|     [xxxx,415]|
+---------------+

火花>=2.2 从Spark 2.2开始（强烈建议使用），您应该改用

多行

选项<代码>多行选项已添加到中