- elasticsearch/
elasticsearch Logstash：映射中字段的顺序错误
elasticsearch Logstash：映射中字段的顺序错误
我有这样一个日志存储配置：
input {
beats {
port => '5044'
}
}
filter {
grok {
patterns_dir => ['./patterns']
match => { 'message' => '%{TIME:timestamp}(","Event\s)(?<Event>([^"]+))(","Form\s)?(?<Form>([^"]+))?(","ParentType\s)?(?<parent_type>([^"]+))?(","ParentName\s)?(?<parent_name>([^"]+))?(","Type\s)?(?<type>([^"]+))?(","Name\s)?(?<Name_of_form>([^"]+))?(","Detail\s)?(?<Detail>([^"]+))?(","t=)?(?<t>([\d]+))?(",")?(?<Status>(end|beg))?' }
add_tag => [ '%{Status}' ]
}
dissect {
mapping => {
'[log][file][path]' => 'C:\Logs\%{login}\%{server}\%{user}\%{filename}.txt'
}
}
date {
match => [ 'timestamp', 'dd.MM.yyyy HH:mm:ss' ]
}
mutate {
add_field => { 'uniqueEvent' => '%{Event}_%{filename}' }
}
elapsed {
unique_id_field => 'uniqueEvent'
start_tag => 'beg'
end_tag => 'end'
new_event_on_match => false
}
if 'elapsed' in [tags] {
aggregate {
task_id => '%{uniqueEvent}'
code => 'map["duration"] = [(event.get("elapsed_time")*1000).to_i]'
map_action => 'create'
}
}
mutate {
remove_field => ['timestamp', 'ecs', 'log', 'tags', 'message', '@version', 'filename', 'input', 'host', 'agent', 't', 'parent_type', 'parent_name', 'type']
rename => {'elapsed_time' => 'Event_duration'}
}
}
output {
elasticsearch {
hosts => ['localhost:9200']
index => 'logs-%{+dd.MM.YYYY}'
}
}
[2020-08-25T19:33:13,365][INFO ][logstash.outputs.elasticsearch][main] Using mapping template from {:path=>"C:\\Users\\moral\\elk_stack\\logstash-7.8.1\\template\\template.json"}
[2020-08-25T19:33:13,406][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"template"=>"logs-*", "version"=>50001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"properties"=>{"@timestamp"=>{"type"=>"date"}, "Event"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Name_of_form"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Detail"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Event_duration"=>{"type"=>"float"}, "elapsed_timestamp_start"=>{"type"=>"date"}, "user"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "login"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "server"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "Status"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}, "uniqueEvent"=>{"type"=>"text", "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}}}}
[2020-08-25T19:33:13,421][INFO ][logstash.outputs.elasticsearch][main] Installing elasticsearch template to _template/log-template
[2020-08-25T19:33:13,468][ERROR][logstash.outputs.elasticsearch][main] Failed to install template. {:message=>"Got response code '400' contacting Elasticsearch at URL 'http://localhost:9200/_template/log-template'", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError", :backtrace=>["C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb:80:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:332:in `perform_request_to_url'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:319:in `block in perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:414:in `with_connection'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:318:in `perform_request'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client/pool.rb:326:in `block in Pool'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:352:in `template_put'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/http_client.rb:86:in `template_install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:28:in `install'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/template_manager.rb:16:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:205:in `install_template'", "C:/Users/moral/elk_stack/logstash-7.8.1/vendor/bundle/jruby/2.5.0/gems/logstash-output-elasticsearch-10.5.1-java/lib/logstash/outputs/elasticsearch/common.rb:49:in `block in setup_after_successful_connection'"]}
{
"template" : "logs-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"properties": {
"@timestamp": {
"type": "date"
},
"Event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Name_of_form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Detail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Event_duration": {
"type": "float"
},
"elapsed_timestamp_start": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"login": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"server": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uniqueEvent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
您需要做的只是告诉Logstash在启动时安装模板。只需确保模板中的
index\u模式logs-*您需要做的只是告诉Logstash在启动时安装模板。只需确保模板中的
index\u模式logs-*不幸的是，它仍然没有成功。添加了错误文本。也许我需要在运行logstash之前手动将其添加到elasticsearch？错误400。。。您是否也可以将template.json文件添加到您的问题中，itAdded template.json中一定有错误。从可能的帮助来看：elasticsearch可以工作，但它完全是空的。你所说的订单到底是什么意思？您看到的源文件将与Logstash发送给ES的完全相同，ES将永远不会修改它。JSON规范也没有提到任何键的顺序，JSON文档是一个hashmap，本质上没有键顺序。是的，JSON文档中没有字段顺序。很高兴你发现了不幸的是它仍然没有成功。添加了错误文本。也许我需要在运行logstash之前手动将其添加到elasticsearch？错误400。。。您是否也可以将template.json文件添加到您的问题中，itAdded template.json中一定有错误。从可能的帮助来看：elasticsearch可以工作，但它完全是空的。你所说的订单到底是什么意思？您看到的源文件将与Logstash发送给ES的完全相同，ES将永远不会修改它。JSON规范也没有提到任何键的顺序，JSON文档是一个hashmap，本质上没有键顺序。是的，JSON文档中没有字段顺序。很高兴你明白了
{
"template" : "logs-*",
"version" : 50001,
"settings" : {
"index.refresh_interval" : "5s"
},
"mappings" : {
"_default_" : {
"properties": {
"@timestamp": {
"type": "date"
},
"Event": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Name_of_form": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Detail": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Event_duration": {
"type": "float"
},
"elapsed_timestamp_start": {
"type": "date"
},
"user": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"login": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"server": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"Status": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uniqueEvent": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
output {
elasticsearch {
hosts => ['localhost:9200']
index => 'logs-%{+dd.MM.YYYY}'
manage_template => true
template_overwrite => true
template_name => "log-template"
template => "/path/to/template.json"
}
}