Apache nifi 将线路协议语句的nifi executeScript写入XDB

Apache nifi 将线路协议语句的nifi executeScript写入XDB,apache-nifi,Apache Nifi,我有一组从kafka主题、mergeContent和ExecuteScript读取json字段并在使用putinfluxDB之前映射到influxDB的过程。我有特定的脚本,因为json消息的格式不同,存在不同的字段和标记。我想创建一个通用脚本,它可以读取json中的任何字段,并将其相应地映射到XDB进程 例如: flowFile = session.get(); if (flowFile != null) { var StreamCallback = Java.type("org.a

我有一组从kafka主题、mergeContent和ExecuteScript读取json字段并在使用putinfluxDB之前映射到influxDB的过程。我有特定的脚本,因为json消息的格式不同,存在不同的字段和标记。我想创建一个通用脚本,它可以读取json中的任何字段,并将其相应地映射到XDB进程

例如:

flowFile = session.get();
if (flowFile != null) {

var StreamCallback = 
   Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"

// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream, 
 outputStream) {
    // Read input FlowFile content
    var type = flowFile.getAttribute('type')
    var time = Number(flowFile.getAttribute('filename'))
    var meta_name = flowFile.getAttribute('meta_name')
    var remote_source_name = flowFile.getAttribute('gethttp.remote.source')

    var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); // 
     message or content
    var message_content = {};
    try {
        message_content = JSON.parse(content);
        transformed_message.state = (((message_content || {}).state || "null"));
        transformed_message.new_records = (((message_content || {}).new_records || 0));
        transformed_message.updated_records = (((message_content || {}).updated_records || 0));
        transformed_message.changed_records = (((message_content || {}).changed_records || 0));
        transformed_message.deleted_records = (((message_content || {}).deleted_records || 0));
        transformed_message.missing_records = (((message_content || {}).missing_records || 0));

        line = line + ",remote_source_name="+ remote_source_name + ",meta_name="+meta_name+",message_type=" + type + " " 
                 + "new_records=" + transformed_message.new_records + ","
                 + "updated_records=" + transformed_message.updated_records + ","
                 + "changed_records=" + transformed_message.changed_records + ","
                 + "deleted_records=" + transformed_message.deleted_records + ","
                 + "missing_records=" + transformed_message.missing_records + " "
                 + time * 1000000 + sep;


        // Write output content
        if (transformed_message) {
            outputStream.write(line.getBytes(StandardCharsets.UTF_8));
        }
    } catch (e) {
        error = true;
        log.error('Something went wrong', e)
        outputStream.write(content.getBytes(StandardCharsets.UTF_8));
    }
}));

if (error) {
    session.transfer(flowFile, REL_FAILURE)
} else {
    session.transfer(flowFile, REL_SUCCESS)
}
}

因此,在上面的脚本中,JSON.parse(内容)行读取这里特别声明的标记。在字段和标记未知的情况下,是否可以使用相同类型的脚本

编辑:

源Json

{"new_records":"1022",
 "updated_records":"546565",
 "changed_records":"546566",
 "deleted_records":"122345",
 "missing_records":"78"}
预期线路协议将流入

"test_measurement",remote_source_name="jmx_k_server",meta_name="fetchRecord",message_type="text"  new_records="1022",updated_records="546565",changed_records="546566","deleted_records"="122345","missing_records"="78" 1265522132
json格式

data = [
{
  “measurement”: MEASUREMENT,
  “tags”: {
       TAGS
  },
  “fields”: {
     FIELDS
   }
 }
 ]

我在研究一个类似的问题时偶然发现了这个问题:

只需在json中循环键。。。(假设您使用的是javascript/ECMAScript)

关键在于:

message_content = JSON.parse(content);
for (key in message_content){
    line += key + "=" + message_content[key] + ","
}
我使用的全剪:

flowFile = session.get();
if (flowFile != null) {

var StreamCallback = 
   Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"

// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream, 
 outputStream) {
    // Read input FlowFile content
    var type = flowFile.getAttribute('type')
    var time = Number(flowFile.getAttribute('filename'))
    var meta_name = flowFile.getAttribute('meta_name')
    var remote_source_name = flowFile.getAttribute('gethttp.remote.source')

    var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); // message or content
    var message_content = {};
    try {
        message_content = JSON.parse(content);
        for (key in message_content){
            if (key == 't') {
                line +=  " " + message_content[key]
            }
            else {          
                line += "," + key + "=" + message_content[key]
            }
        }
        line += sep

        // Write output content
        if (transformed_message) {
            outputStream.write(line.getBytes(StandardCharsets.UTF_8));
        }
    } catch (e) {
        error = true;
        log.error('Something went wrong', e)
        outputStream.write(content.getBytes(StandardCharsets.UTF_8));
    }
}));

if (error) {
    session.transfer(flowFile, REL_FAILURE)
} else {
    session.transfer(flowFile, REL_SUCCESS)
}
}

请提供源json消息和目标json的示例。