使用logstash中的grock将nginx日志转换为elasticsearch

我收到nginx的以下消息：

XXX.XXX.XXX.XXX - - [09/Jul/2014:15:23:51 +0200] "GET /js/order.js HTTP/1.1" 200 2777 "http://www.yyy.xxx.zz/accueil"; "Mozilla/5.0 (Windows NT 5.1; rv:30.0) Gecko/20100101 Firefox/30.0" "-" "0.000" "317" 为了实现这一点，我使用以下模式：

match => [ "message", "%{IPORHOST:clientip} - - \[%{log-date:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER\:response} (?:%{NUMBER:bytes}|-) \"%{URI:url}" ]

我遇到的问题是，所有字段都被视为字符串：

"method"   :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"offset"   :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"request"  :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"response" :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"timestamp":{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"type"     :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}},
"url"      :{"type":"string","norms":{"enabled":false},"fields":{"raw":{"type":"string","index":"not_analyzed","ignore_above":256}}}

我想将timestamp字段设置为“date”，甚至更好，用这个时间戳替换@timestamp，以便能够用elasticsearch的rangeapi查询它

根据Alcanzar的建议，我修改了conf文件如下：

input {
  lumberjack {
    # The port to listen on                                                                                                                                  
    port => 5140

    # The paths to your ssl cert and key                                                                                                                     
    ssl_certificate => "/etc/logstash/logstash.crt"
    ssl_key => "/etc/logstash/logstash.key"

    # Set this to whatever you want.                                                                                                                         
    type => "nginx"
  }
}

filter {
  grok {
    match => [ "message", "%{IPORHOST:clientip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER\
:response} (?:%{NUMBER:bytes}|-) \"%{URI:url}" ]
  }
}

filter {
 date { match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] }
}

output {
  elasticsearch {
    host => localhost
    index => front_gpp3
  }
  stdout { codec => rubydebug }
}

问题在于区域设置，我的VM将FR作为区域设置而不是EN，因此我添加了以下内容以将其添加到筛选器中：

filter {
  grok {
    match => [ "message", "%{IPORHOST:clientip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER\
:response} (?:%{NUMBER:bytes}|-) \"%{URI:url}" ]
  }
  date {
    locale => "en"
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}

您需要添加一个过滤器：

或者类似的东西。这将使用该格式解析

时间戳

，然后使用解析后的值设置

@timestamp

filter {
  grok {
    match => [ "message", "%{IPORHOST:clientip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER\
:response} (?:%{NUMBER:bytes}|-) \"%{URI:url}" ]
  }
  date {
    locale => "en"
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}

filter {
 date { match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ] }
}