Scala 使用Spark将json映射到case类（字段名中的空格）

我试图用spark

Dataset

API读取一个json文件，问题是这个json在一些字段名中包含空格

这将是一个json行

{"Field Name" : "value"}

我的案例课应该是这样的

case class MyType(`Field Name`: String)

然后，我可以将该文件加载到

数据帧中，它将加载正确的模式

val dataframe = spark.read.json(path)

当我尝试将
DataFrame
转换为
Dataset[MyType]

dataframe.as[MyType]

编码器[MyType]
加载的StructSchema是错误的，它引入了
$u0020
而不是空格，我得到以下错误

cannot resolve '`Field$u0020Name`' given input columns: [Field Name];
org.apache.spark.sql.AnalysisException: cannot resolve '`Field$u0020Name`' given input columns: [Field Name];
    at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:88)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1$$anonfun$apply$2.applyOrElse(CheckAnalysis.scala:85)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:289)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:289)
    at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:288)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4$$anonfun$apply$11.apply(TreeNode.scala:335)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
    at scala.collection.immutable.List.map(List.scala:285)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:333)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:286)
    at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply(QueryPlan.scala:268)
    at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply(QueryPlan.scala:268)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:279)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1(QueryPlan.scala:289)
    at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$6.apply(QueryPlan.scala:298)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:298)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:268)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:85)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:78)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:127)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:78)
    at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:91)
    at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.resolveAndBind(ExpressionEncoder.scala:256)
    at org.apache.spark.sql.Dataset.<init>(Dataset.scala:206)
    at org.apache.spark.sql.Dataset.<init>(Dataset.scala:170)
    at org.apache.spark.sql.Dataset$.apply(Dataset.scala:61)
    at org.apache.spark.sql.Dataset.as(Dataset.scala:380)
    at com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply$mcV$sp(BomboraSuite.scala:151)
    at com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply(BomboraSuite.scala:141)
    at com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply(BomboraSuite.scala:141)
    at org.scalatest.OutcomeOf$class.outcomeOf(OutcomeOf.scala:85)
    at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
    at org.scalatest.Transformer.apply(Transformer.scala:22)
    at org.scalatest.Transformer.apply(Transformer.scala:20)
    at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
    at org.scalatest.TestSuite$class.withFixture(TestSuite.scala:196)
    at org.scalatest.FunSuite.withFixture(FunSuite.scala:1560)
    at org.scalatest.FunSuiteLike$class.invokeWithFixture$1(FunSuiteLike.scala:183)
    at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
    at org.scalatest.FunSuiteLike$$anonfun$runTest$1.apply(FunSuiteLike.scala:196)
    at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
    at org.scalatest.FunSuiteLike$class.runTest(FunSuiteLike.scala:196)
    at org.scalatest.FunSuite.runTest(FunSuite.scala:1560)
    at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
    at org.scalatest.FunSuiteLike$$anonfun$runTests$1.apply(FunSuiteLike.scala:229)
    at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:396)
    at org.scalatest.SuperEngine$$anonfun$traverseSubNodes$1$1.apply(Engine.scala:384)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
    at org.scalatest.SuperEngine.org$scalatest$SuperEngine$$runTestsInBranch(Engine.scala:379)
    at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
    at org.scalatest.FunSuiteLike$class.runTests(FunSuiteLike.scala:229)
    at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
    at org.scalatest.Suite$class.run(Suite.scala:1147)
    at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
    at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
    at org.scalatest.FunSuiteLike$$anonfun$run$1.apply(FunSuiteLike.scala:233)
    at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
    at org.scalatest.FunSuiteLike$class.run(FunSuiteLike.scala:233)
    at com.radius.floodgate.preprocess.BomboraSuite.org$scalatest$BeforeAndAfterAll$$super$run(BomboraSuite.scala:18)
    at org.scalatest.BeforeAndAfterAll$class.liftedTree1$1(BeforeAndAfterAll.scala:213)
    at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:210)
    at com.radius.floodgate.preprocess.BomboraSuite.run(BomboraSuite.scala:18)
    at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
    at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1340)
    at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$1.apply(Runner.scala:1334)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1334)
    at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1011)
    at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1010)
    at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1500)
    at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1010)
    at org.scalatest.tools.Runner$.run(Runner.scala:850)
    at org.scalatest.tools.Runner.run(Runner.scala)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28)

无法解析给定输入列“`Field$u0020Name`:[字段名]；
org.apache.spark.sql.AnalysisException:无法解析给定输入列的“`Field$u0020Name`:[Field Name]；
位于org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis（package.scala:42）
在org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$CheckAnalysis$1$$anonfun$apply$2.applyOrElse（CheckAnalysis.scala:88）
在org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$CheckAnalysis$1$$anonfun$apply$2.applyOrElse（CheckAnalysis.scala:85）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply（TreeNode.scala:289）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply（TreeNode.scala:289）
位于org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin（TreeNode.scala:70）
位于org.apache.spark.sql.catalyst.trees.TreeNode.transformUp（TreeNode.scala:288）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply（TreeNode.scala:306）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator（TreeNode.scala:187）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren（TreeNode.scala:304）
位于org.apache.spark.sql.catalyst.trees.TreeNode.transformUp（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply（TreeNode.scala:306）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator（TreeNode.scala:187）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren（TreeNode.scala:304）
位于org.apache.spark.sql.catalyst.trees.TreeNode.transformUp（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4$$anonfun$apply$11.apply（TreeNode.scala:335）
在scala.collection.TraversableLike$$anonfun$map$1.apply处（TraversableLike.scala:234）
在scala.collection.TraversableLike$$anonfun$map$1.apply处（TraversableLike.scala:234）
位于scala.collection.immutable.List.foreach（List.scala:381）
位于scala.collection.TraversableLike$class.map（TraversableLike.scala:234）
位于scala.collection.immutable.List.map（List.scala:285）
位于org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply（TreeNode.scala:333）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator（TreeNode.scala:187）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren（TreeNode.scala:304）
位于org.apache.spark.sql.catalyst.trees.TreeNode.transformUp（TreeNode.scala:286）
位于org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply（QueryPlan.scala:268）
位于org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$transformExpressionsUp$1.apply（QueryPlan.scala:268）
位于org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1（QueryPlan.scala:279）
在org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$recursiveTransform$1（QueryPlan.scala:289）
在org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$6.apply上（QueryPlan.scala:298）
位于org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator（TreeNode.scala:187）
位于org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions（QueryPlan.scala:298）
位于org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp（QueryPlan.scala:268）
位于org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$CheckAnalysis$1.apply（CheckAnalysis.scala:85）
位于org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$CheckAnalysis$1.apply（CheckAnalysis.scala:78）
位于org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp（TreeNode.scala:127）
位于org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.CheckAnalysis（CheckAnalysis.scala:78）
位于org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis（Analyzer.scala:91）
位于org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.resolveAndBind（ExpressionEncoder.scala:256）
位于org.apache.spark.sql.Dataset。（Dataset.scala:206）
位于org.apache.spark.sql.Dataset（Dataset.scala:170）
位于org.apache.spark.sql.Dataset$.apply（Dataset.scala:61）
位于org.apache.spark.sql.Dataset.as（Dataset.scala:380）
在com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply$mcV$sp（BomboraSuite.scala:151）
位于com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply（BomboraSuite.scala:141）
位于com.radius.floodgate.preprocess.BomboraSuite$$anonfun$5.apply（BomboraSuite.scala:141）
在org.scalatest.OutcomeOf$class.OutcomeOf（OutcomeOf.scala:85）
在org.scalatest.OutcomeOf$.OutcomeOf（OutcomeOf.scala:104）
在org.scalatest.Transformer.apply上（Transformer.scala:22）
在org.scalatest.Transformer.apply上（Transformer.scala:20）
在org.scalatest.FunSuiteLike$$anon$1.apply上（FunSuiteLike.scala:186）
在org.scalatest.TestSuite$class.withFixture（Te
case class MyType(Field_Name: String)

dataframe.withColumnRenamed("Field Name", "Field_Name").as[MyType]