在JAVA中将复杂的嵌套Json转换为Spark数据帧
有人能帮我用Java代码将下面的JSON转换成Spark Dataframe吗 注意：它不是文件 逻辑： 听卡夫卡主题T1，读取RDD中的每条记录，并应用附加逻辑将结果数据转换为Json对象，然后将其写入卡夫卡中的另一个主题T2 T2结构如下 JSON:在JAVA中将复杂的嵌套Json转换为Spark数据帧,java,json,apache-spark,dataframe,nested,Java,Json,Apache Spark,Dataframe,Nested,有人能帮我用Java代码将下面的JSON转换成Spark Dataframe吗 注意：它不是文件 逻辑： 听卡夫卡主题T1，读取RDD中的每条记录，并应用附加逻辑将结果数据转换为Json对象，然后将其写入卡夫卡中的另一个主题T2 T2结构如下 JSON: [ { "@tenant_id":"XYZ", "alarmUpdateTime":1526342400000, "alarm_id":"AB5C9123", "alarm_upd
[
{
"@tenant_id":"XYZ",
"alarmUpdateTime":1526342400000,
"alarm_id":"AB5C9123",
"alarm_updates":[
{
"alarmField":"Severity",
"new_value":"Minor",
"old_value":"Major"
},
{
"alarmField":"state",
"new_value":"UPDATE",
"old_value":"NEW"
}
],
"aucID":"5af83",
"inID":"INC15234567",
"index":"test",
"product":"test",
"source":"ABS",
"state":"NEW"
}
]
ClassAlarm{
String @tenant_id;
String alarm_id;
.
.
List <AlarmUpdate> update;
Get and Setter functions for all variables
}
AlarmUpdate{
String alarmField;
String oldVal;
String NewVal;
Get and Setter functions for all variables
}
AppClass{
void static main(){
Alarm alarmObj = new Alarm();
//set values for variables in alarmObj.
Dataset <Row> results = jobCtx.getSparkSession().createDataFrame(Arrays.asList(alarmObj), Alarm.class)
//At this point seeing following errors.
}
}
ClassAlarm{
字符串@tenant_id；
字符串报警标识；
.
.
名单更新；
所有变量的Get和Setter函数
}
警报更新{
字符串报警字段；
字符串oldVal；
字符串NewVal；
所有变量的Get和Setter函数
}
应用类{
void static main（）{
报警对象=新报警（）；
//为alarmObj中的变量设置值。
Dataset results=jobCtx.getSparkSession（）.createDataFrame（Arrays.asList（alarmObj）、Alarm.class）
//此时会看到以下错误。
}
}
import org.apache.spark.sql.SparkSession;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
static SparkSession spark = SparkSession.builder().master("local").appName("simple").getOrCreate();
static JavaSparkContext sc = new JavaSparkContext(spark.sparkContext());
Dataset<Row> df = spark.read().json(sc.wholeTextFiles("path to json file").map(t -> t._2()));
df.show(false);
+----------+---------------+--------+--------------------------------------------+-----+-----------+-----+-------+------+-----+
|@tenant_id|alarmUpdateTime|alarm_id|alarm_updates |aucID|inID |index|product|source|state|
+----------+---------------+--------+--------------------------------------------+-----+-----------+-----+-------+------+-----+
|XYZ |1526342400000 |AB5C9123|[[Severity,Minor,Major], [state,UPDATE,NEW]]|5af83|INC15234567|test |test |ABS |NEW |
+----------+---------------+--------+--------------------------------------------+-----+-----------+-----+-------+------+-----+
String t1Record = "[\n" +
" {\n" +
" \"@tenant_id\":\"XYZ\",\n" +
" \"alarmUpdateTime\":1526342400000,\n" +
" \"alarm_id\":\"AB5C9123\",\n" +
" \"alarm_updates\":[\n" +
" {\n" +
" \"alarmField\":\"Severity\",\n" +
" \"new_value\":\"Minor\",\n" +
" \"old_value\":\"Major\"\n" +
" },\n" +
" {\n" +
" \"alarmField\":\"state\",\n" +
" \"new_value\":\"UPDATE\",\n" +
" \"old_value\":\"NEW\"\n" +
" }\n" +
" ],\n" +
" \"aucID\":\"5af83\",\n" +
" \"inID\":\"INC15234567\",\n" +
" \"index\":\"test\",\n" +
" \"product\":\"test\",\n" +
" \"source\":\"ABS\",\n" +
" \"state\":\"NEW\"\n" +
" }\n" +
"]";
JavaRDD<String> t1RecordRDD = sc.parallelize(Arrays.asList(t1Record));
Dataset<Row> df = spark.read().json(t1RecordRDD);