Apache nifi 将线路协议语句的nifi executeScript写入XDB
我有一组从kafka主题、mergeContent和ExecuteScript读取json字段并在使用putinfluxDB之前映射到influxDB的过程。我有特定的脚本,因为json消息的格式不同,存在不同的字段和标记。我想创建一个通用脚本,它可以读取json中的任何字段,并将其相应地映射到XDB进程 例如:Apache nifi 将线路协议语句的nifi executeScript写入XDB,apache-nifi,Apache Nifi,我有一组从kafka主题、mergeContent和ExecuteScript读取json字段并在使用putinfluxDB之前映射到influxDB的过程。我有特定的脚本,因为json消息的格式不同,存在不同的字段和标记。我想创建一个通用脚本,它可以读取json中的任何字段,并将其相应地映射到XDB进程 例如: flowFile = session.get(); if (flowFile != null) { var StreamCallback = Java.type("org.a
flowFile = session.get();
if (flowFile != null) {
var StreamCallback =
Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"
// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream,
outputStream) {
// Read input FlowFile content
var type = flowFile.getAttribute('type')
var time = Number(flowFile.getAttribute('filename'))
var meta_name = flowFile.getAttribute('meta_name')
var remote_source_name = flowFile.getAttribute('gethttp.remote.source')
var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); //
message or content
var message_content = {};
try {
message_content = JSON.parse(content);
transformed_message.state = (((message_content || {}).state || "null"));
transformed_message.new_records = (((message_content || {}).new_records || 0));
transformed_message.updated_records = (((message_content || {}).updated_records || 0));
transformed_message.changed_records = (((message_content || {}).changed_records || 0));
transformed_message.deleted_records = (((message_content || {}).deleted_records || 0));
transformed_message.missing_records = (((message_content || {}).missing_records || 0));
line = line + ",remote_source_name="+ remote_source_name + ",meta_name="+meta_name+",message_type=" + type + " "
+ "new_records=" + transformed_message.new_records + ","
+ "updated_records=" + transformed_message.updated_records + ","
+ "changed_records=" + transformed_message.changed_records + ","
+ "deleted_records=" + transformed_message.deleted_records + ","
+ "missing_records=" + transformed_message.missing_records + " "
+ time * 1000000 + sep;
// Write output content
if (transformed_message) {
outputStream.write(line.getBytes(StandardCharsets.UTF_8));
}
} catch (e) {
error = true;
log.error('Something went wrong', e)
outputStream.write(content.getBytes(StandardCharsets.UTF_8));
}
}));
if (error) {
session.transfer(flowFile, REL_FAILURE)
} else {
session.transfer(flowFile, REL_SUCCESS)
}
}
因此,在上面的脚本中,JSON.parse(内容)行读取这里特别声明的标记。在字段和标记未知的情况下,是否可以使用相同类型的脚本
编辑:
源Json
{"new_records":"1022",
"updated_records":"546565",
"changed_records":"546566",
"deleted_records":"122345",
"missing_records":"78"}
预期线路协议将流入
"test_measurement",remote_source_name="jmx_k_server",meta_name="fetchRecord",message_type="text" new_records="1022",updated_records="546565",changed_records="546566","deleted_records"="122345","missing_records"="78" 1265522132
json格式
data = [
{
“measurement”: MEASUREMENT,
“tags”: {
TAGS
},
“fields”: {
FIELDS
}
}
]
我在研究一个类似的问题时偶然发现了这个问题: 只需在json中循环键。。。(假设您使用的是javascript/ECMAScript) 关键在于:
message_content = JSON.parse(content);
for (key in message_content){
line += key + "=" + message_content[key] + ","
}
我使用的全剪:
flowFile = session.get();
if (flowFile != null) {
var StreamCallback =
Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"
// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream,
outputStream) {
// Read input FlowFile content
var type = flowFile.getAttribute('type')
var time = Number(flowFile.getAttribute('filename'))
var meta_name = flowFile.getAttribute('meta_name')
var remote_source_name = flowFile.getAttribute('gethttp.remote.source')
var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); // message or content
var message_content = {};
try {
message_content = JSON.parse(content);
for (key in message_content){
if (key == 't') {
line += " " + message_content[key]
}
else {
line += "," + key + "=" + message_content[key]
}
}
line += sep
// Write output content
if (transformed_message) {
outputStream.write(line.getBytes(StandardCharsets.UTF_8));
}
} catch (e) {
error = true;
log.error('Something went wrong', e)
outputStream.write(content.getBytes(StandardCharsets.UTF_8));
}
}));
if (error) {
session.transfer(flowFile, REL_FAILURE)
} else {
session.transfer(flowFile, REL_SUCCESS)
}
}
请提供源json消息和目标json的示例。