elasticsearch 使用Logstash分析对象数组,elasticsearch,logstash,logstash-configuration,elasticsearch,Logstash,Logstash Configuration" /> elasticsearch 使用Logstash分析对象数组,elasticsearch,logstash,logstash-configuration,elasticsearch,Logstash,Logstash Configuration" />

elasticsearch 使用Logstash分析对象数组

elasticsearch 使用Logstash分析对象数组,elasticsearch,logstash,logstash-configuration,elasticsearch,Logstash,Logstash Configuration,我有JMS消息负载,其中包含需要索引到弹性搜索中的客户对象数组 JMS有效负载 { "file": { "header": { "id": "13", "name": "ElasticCustomer", "description": "ElasticCustomer", &qu

我有JMS消息负载,其中包含需要索引到弹性搜索中的客户对象数组

JMS有效负载

{
  "file": {
    "header": {
      "id": "13",
      "name": "ElasticCustomer",
      "description": "ElasticCustomer",
      "version": "Version 1",
      "exportID": "ElasticCustomer",
      "exportInstance": "20200918043921264",
      "extractionMode": "FULL",
      "extractionDate": "2020-09-18T05:37:32"
    },
    "body": {
      "records": [
        {
          "customerId": "NVS1200063862",
          "URI": "https://data.novartis.net/r1/commercial/customer_management/customer_mdm/v1/customer/HCO/NVS1200063862",
          "customerCountry": "200024",
          "type": {
            "linked_record": {
              "code": "200027",
              "description": "Health Care Organization",
              "shortDescription": "HCO",
              "remoteKeys": {
                "linked_records": []
              },
              "technicalData": {
                "lastUser": "chaoure1",
                "creationTime": "2019-06-24T11:02:45",
                "lastTime": "2019-06-24T11:02:45",
                "primaryKey": "200027",
                "foreignKey": "200027",
                "linkedTable": "/root/ReferenceData"
              }
            }
          },
          "veevaRecordType": {
            "recordTypeId": "012A0000000zC10IAE",
            "recordTypeValue": "PH_CORE-Clinics"
          },
          "status": {
            "linked_record": {
              "code": "200036",
              "description": "Active",
              "shortDescription": null,
              "remoteKeys": {
                "linked_records": [
                  {
                    "system": "200073",
                    "systemKey": "STA.3",
                    "technicalData": {
                      "lastUser": "moritda3",
                      "creationTime": "2020-04-22T19:48:08",
                      "lastTime": "2020-04-22T19:48:08",
                      "primaryKey": "6099",
                      "linkedTable": "/root/RemoteKeys"
                    }
                  },
                  {
                    "system": "200072",
                    "systemKey": "Active",
                    "technicalData": {
                      "lastUser": "moritda3",
                      "creationTime": "2020-04-22T19:48:08",
                      "lastTime": "2020-04-22T19:48:08",
                      "primaryKey": "6351",
                      "linkedTable": "/root/RemoteKeys"
                    }
                  },
                  {
                    "system": "200071",
                    "systemKey": "Active",
                    "technicalData": {
                      "lastUser": "moritda3",
                      "creationTime": "2020-04-22T19:48:08",
                      "lastTime": "2020-04-22T19:48:08",
                      "primaryKey": "6353",
                      "linkedTable": "/root/RemoteKeys"
                    }
                  }
                ]
              },
              "technicalData": {
                "lastUser": "chaoure1",
                "creationTime": "2019-06-24T11:02:45",
                "lastTime": "2019-06-24T11:02:45",
                "primaryKey": "200036",
                "foreignKey": "200036",
                "linkedTable": "/root/ReferenceData"
              }
            }
          },
          "statusReason": {
            "linked_record": {}
          },
          "hco": "HCO1200016052",
          "hcp": null,
          "healthCareOrganisation": {
            "linked_records": []
          },
          "healthCareProfessional": {
            "linked_records": []
          },
          "communication": {
            "linked_records": [
              {
                "communicationId": 2739472,
                "customerId": "NVS1200063862",
                "communicationType": {
                  "linked_record": {
                    "code": "200146",
                    "description": "Communication Channel",
                    "shortDescription": null,
                    "remoteKeys": {
                      "linked_records": []
                    },
                    "technicalData": {
                      "lastUser": "chaoure1",
                      "creationTime": "2019-06-24T11:02:45",
                      "lastTime": "2019-06-24T11:02:45",
                      "primaryKey": "200146",
                      "foreignKey": "200146",
                      "linkedTable": "/root/ReferenceData"
                    }
                  }
                },
                "type": "212126",
                "value": "519 2046052",
                "status": {
                  "linked_record": {
                    "code": "200084",
                    "description": "Active",
                    "shortDescription": null,
                    "remoteKeys": {
                      "linked_records": []
                    },
                    "technicalData": {
                      "lastUser": "chaoure1",
                      "creationTime": "2019-06-24T11:02:45",
                      "lastTime": "2019-06-24T11:02:45",
                      "primaryKey": "200084",
                      "foreignKey": "200084",
                      "linkedTable": "/root/ReferenceData"
                    }
                  }
                },
                "isPrimary": false,
                "technicalData": {
                  "lastUser": "sys_apimdmprod",
                  "creationTime": "2020-08-07T08:01:30",
                  "lastTime": "2020-08-07T08:01:30",
                  "primaryKey": "2739472",
                  "linkedTable": "/root/Communication"
                }
              }
            ]
          },
          "remoteKeys": {
            "linked_records": []
          },
          "adresses": {
            "linked_records": [
              {
                "addressId": 6748337,
                "Identification": {
                  "customerId": "NVS1200063862",
                  "isPrimary": true,
                  "addressType": {
                    "linked_record": {
                      "code": "200148",
                      "description": "Primary address",
                      "shortDescription": null,
                      "remoteKeys": {
                        "linked_records": [
                          {
                            "system": "200072",
                            "systemKey": "Primary address",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "5412",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          },
                          {
                            "system": "200071",
                            "systemKey": "Primary address",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "5417",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          },
                          {
                            "system": "200073",
                            "systemKey": "TYS.P",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "5422",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          }
                        ]
                      },
                      "technicalData": {
                        "lastUser": "chaoure1",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2019-06-24T11:02:45",
                        "primaryKey": "200148",
                        "foreignKey": "200148",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  }
                },
                "AddressAttributes": {
                  "addressLine1": "D-590 Oxford St E",
                  "addressLine2": "D",
                  "addressLine3": null,
                  "city": "London",
                  "stateandprovince": {
                    "linked_record": {
                      "code": "213040",
                      "description": "ON",
                      "shortDescription": "ON",
                      "remoteKeys": {
                        "linked_records": [
                          {
                            "code": 6672,
                            "system": "200071",
                            "systemKey": "ON",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "6672",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          },
                          {
                            "code": 6685,
                            "system": "200073",
                            "systemKey": "009",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "6685",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          },
                          {
                            "code": 11365,
                            "system": "209336",
                            "systemKey": "ON",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "11365",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          }
                        ]
                      },
                      "technicalData": {
                        "lastUser": "chaoure1",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2019-06-24T11:02:45",
                        "primaryKey": "213040",
                        "foreignKey": "213040",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  },
                  "zip": "N5Y 3J1",
                  "country": {
                    "linked_record": {
                      "code": "200024",
                      "description": "Canada",
                      "shortDescription": "CA",
                      "remoteKeys": {
                        "linked_records": [
                          {
                            "system": "200071",
                            "systemKey": "Canada",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "6081",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          },
                          {
                            "system": "200073",
                            "systemKey": "CA",
                            "status": "AC",
                            "technicalData": {
                              "lastUser": "moritda3",
                              "creationTime": "2020-04-22T19:48:08",
                              "lastTime": "2020-04-22T19:48:08",
                              "primaryKey": "6082",
                              "linkedTable": "/root/RemoteKeys"
                            }
                          }
                        ]
                      },
                      "technicalData": {
                        "lastUser": "singhv2z",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2020-05-04T06:48:38",
                        "primaryKey": "200024",
                        "foreignKey": "200024",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  }
                },
                "AdditionalAttributes": {
                  "brick": "N5Y",
                  "microbrick": null,
                  "latitude": null,
                  "longitude": null,
                  "height": null,
                  "geographicCoordinates": null,
                  "masterAddress": null,
                  "effectiveDate": "2020-08-06",
                  "addressStatus": {
                    "linked_record": {
                      "code": "200003",
                      "description": "Active",
                      "shortDescription": null,
                      "remoteKeys": {
                        "linked_records": []
                      },
                      "technicalData": {
                        "lastUser": "chaoure1",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2019-06-24T11:02:45",
                        "primaryKey": "200003",
                        "foreignKey": "200003",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  },
                  "addressVerificationStatus": {
                    "linked_record": {
                      "code": "200032",
                      "description": "Verified",
                      "shortDescription": null,
                      "remoteKeys": {
                        "linked_records": []
                      },
                      "technicalData": {
                        "lastUser": "chaoure1",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2019-06-24T11:02:45",
                        "primaryKey": "200032",
                        "foreignKey": "200032",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  }
                },
                "technicalData": {
                  "lastUser": "sys_apimdmprod",
                  "creationTime": "2020-08-07T08:01:35",
                  "lastTime": "2020-08-07T08:01:35",
                  "primaryKey": "6748337",
                  "linkedTable": "/root/Address"
                }
              }
            ]
          },
          "education": {
            "linked_records": []
          },
          "specialties": {
            "linked_records": []
          },
          "consent": {
            "linked_records": []
          },
          "credentials": {
            "linked_records": []
          },
          "Affiliation": {
            "customerParents": {
              "linked_records": [
                {
                  "affiliationId": 5977284,
                  "parentId": "NVS1200063862",
                  "childId": "NVS1000143585",
                  "affiliationType": {
                    "linked_record": {
                      "code": "200006",
                      "description": "HCO-HCP",
                      "shortDescription": null,
                      "remoteKeys": {
                        "linked_records": []
                      },
                      "technicalData": {
                        "lastUser": "chaoure1",
                        "creationTime": "2019-06-24T11:02:45",
                        "lastTime": "2019-06-24T11:02:45",
                        "primaryKey": "200006",
                        "foreignKey": "200006",
                        "linkedTable": "/root/ReferenceData"
                      }
                    }
                  },
                  "startDate": "2020-08-07",
                  "endDate": null,
                  "isPrimary": true,
                  "nanobrick": null,
                  "role": null,
                  "technicalData": {
                    "lastUser": "sys_apimdmprod",
                    "creationTime": "2020-08-07T08:01:55",
                    "lastTime": "2020-08-07T08:01:55",
                    "primaryKey": "5977284",
                    "linkedTable": "/root/Affiliation"
                  }
                }
              ]
            },
            "customerChild": {
              "linked_records": []
            }
          },
          "additionalQualification": {
            "linked_records": []
          },
          "statusComment": null,
          "Tracking": {
            "technicalRecordStatus": "PU",
            "producerSystem": "OneKey",
            "producerSystemRecordID": "WCAH00083151",
            "transactionID": "WORKPLACE_execution3ab77e11c84a403aa12a33c53fafaa4520200807080051163",
            "createdDate": "2020-08-07T08:01:27",
            "createdBy": "OneKey",
            "LastUpdatedDate": "2020-08-07T08:01:35",
            "lastUpdateBy": "Api Gateway (sys_apimdmprod)",
            "ApprovalDate": null,
            "ApprovedBy": null,
            "workflowID": null
          },
          "technicalData": {
            "lastUser": "ebx-systemUser",
            "creationTime": "2020-08-07T08:01:29",
            "lastTime": "2020-09-17T15:41:04",
            "primaryKey": "NVS1200063862"
          }
        }
      ]
    },
    "footer": {
      "nb_records": 1
    }
  }
}
我想为
records
数组字段中的每条记录创建单独的upsert logstash事件

我当前的日志存储配置如下所示

input
{
    jms {
        include_header => false
        include_properties => false
        include_body => true
        use_jms_timestamp => false
        destination => 'SpringBatchTestQueue'
        pub_sub => false
        jndi_name => '/JMS/CF/MDM'
        jndi_context => {
         'java.naming.factory.initial' => 'com.solacesystems.jndi.SolJNDIInitialContextFactory'
         'java.naming.security.principal' => 'EDM_Test_User@NovartisDevVPN'
         'java.naming.provider.url' => 'tcp://localhost:55555'
         'java.naming.security.credentials' => 'EDM_Test_User'
        }
        require_jars=> ['/app/elasticsearch/jms/commons-lang-2.6.jar',
                        '/app/elasticsearch/jms/sol-jms-10.10.0.jar',
                        '/app/elasticsearch/jms/geronimo-jms_1.1_spec-1.1.1.jar']
    }
}
filter{
    json{
        source => "message"
    }
split {
field => "[file][body][records]"
}
}

output
{
    elasticsearch
    {
        hosts => ["https://localhost:9200/"]
        index => "test-%{+YYYY.MM.dd}"
        document_id => "%{customerId}"
        doc_as_upsert => true
        ssl => true
        ssl_certificate_verification => true
        cacert => "/app/elasticsearch/config/ssl/Novartis_Silver_Three_Chain.pem"
    }
}
有人能告诉我如何使用logstash将
记录
字段中的每个
customerId
索引到ES中吗

编辑1:根据Alcanzer建议的答案。
\u id
字段未正确填充,消息也包含不必要的字段

您需要使用在
[文件][正文][记录]
上进行拆分。拆分过滤器的一个用例是获取一个数组,并为数组中的每个元素创建一个事件

你只要加上

split { field => "[file][body][records]" } 

在您的
json
过滤器之后。

它必须是
[文件][正文][记录]
?由于
文件
包含
[正文][记录]
?是。。。将修复答案不起作用..您可以检查更新的问题吗?您的文档id字段应为
%{[file][body][records][customer\u id]}