Logstash _带麋鹿的沼泽地

我是编程/Linux/ELK等的新手。我的背景是Windows，所以这个项目对我来说是一个巨大的飞跃

我似乎已经到了一个无法克服的地步，我想用另一双眼睛来回顾我的工作

在Kibana 3中查看输出时，所有自定义字段返回为空，即使在logstash rubydebug中它们显示为已填充。请参见下面的rubydebug输出：

 "message" => "<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454722] ",
          "@version" => "1",
        "@timestamp" => "2015-03-23T21:46:49.000Z",
              "host" => "1.1.1.1",
    "rsyslogprepend" => "<158>Mar 23 16:46:52 servername server-log",
         "timestamp" => "Mon Mar 23 16:46:49 2015",
             "bon01" => "43227.23454683",
          "username" => "dummy.user",
         "ipaddress" => [
        [0] "2.2.2.2",
        [1] "2.2.2.2"
    ],
             "bon02" => "23454722"
}
filter received {:event=>{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, :level=>:debug, :file=>"(eval)", :line=>"24"}
Running grok filter {:event=>#<LogStash::Event:0x370ea56c @accessors=#<LogStash::Util::Accessors:0x228e71b1 @store={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, @lut={"host"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, "host"]}>, @data={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1"}, @cancelled=false>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"280"}
Event now:  {:event=>#<LogStash::Event:0x370ea56c @accessors=#<LogStash::Util::Accessors:0x228e71b1 @store={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, @lut={"host"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "host"], "message"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "message"], "rsyslogprepend"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "rsyslogprepend"], "timestamp"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "timestamp"], "bon01"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "bon01"], "username"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "username"], "ipaddress"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "ipaddress"], "bon02"=>[{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, "bon02"]}>, @data={"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T16:46:52.448Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, @cancelled=false>, :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"300"}
Date filter: received event {:type=>nil, :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"178"}
Date filter looking for field {:type=>nil, :field=>"timestamp", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"181"}
Date parsing done {:value=>"Mon Mar 23 16:46:49 2015", :level=>:debug, :file=>"logstash/filters/date.rb", :line=>"210"}
output received {:event=>{"message"=>"<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ", "@version"=>"1", "@timestamp"=>"2015-03-23T21:46:49.000Z", "host"=>"1.1.1.1", "rsyslogprepend"=>"<158>Mar 23 16:46:52 servername server-log", "timestamp"=>"Mon Mar 23 16:46:49 2015", "bon01"=>"43227.23454683", "username"=>"dummy.user", "ipaddress"=>["1.1.1.1", "1.1.1.1"], "bon02"=>"23454723"}, :level=>:debug, :file=>"(eval)", :line=>"57"}
{
           "message" => "<158>Mar 23 16:46:52 servername server-log [Mon Mar 23 16:46:49 2015][43227.23454683] user dummy.user : testing 1.1.1.1 (1.1.1.1) [23454723] ",
          "@version" => "1",
        "@timestamp" => "2015-03-23T21:46:49.000Z",
              "host" => "1.1.1.1",
    "rsyslogprepend" => "<158>Mar 23 16:46:52 servername server-log",
         "timestamp" => "Mon Mar 23 16:46:49 2015",
             "bon01" => "43227.23454683",
          "username" => "dummy.user",
         "ipaddress" => [
        [0] "1.1.1.1",
        [1] "1.1.1.1"
    ],
             "bon02" => "23454723"
}


logstash conf file below:

    # syslog input

    input {

     tcp {
        port => 514
        #type => syslog
      }
      udp {
        port => 514
       #type => syslog
      }
    }

    filter {
                    grok {
                                    patterns_dir => "opt/logstash/patterns"

                                    #       match => [ "message", "%{NESSUS_MUTATE_RSYSLOG:syslog_prepend}" ]
    #       remove_field => [ "syslog_prepend" ]

    #               }

    #                mutate {
    #        remove_field => [ "syslog_prepend" ]
    #                }

    #               grok {

    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} : testing %{IPV4:ipaddress} \(%{IPV4:ipaddress}\) \[%{NUMBER:bon02}\]"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} : The remote host \(%{IPV4:ipaddress}\) is dead"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] \[nessusd_www_server\] User %{USERNAME:username} \(%{IPV4:ipaddress}\) successfully logged out"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] \[nessusd_www_server\] successful login of \'%{USERNAME:username}\' from %{IPV4:ipaddress} via %{NESSUS_PROTOCOL:protocol}"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] Finished testing %{IPV4:ipaddress}. Time : %{NESSUS_DURATION:duration}"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] User \'%{USERNAME:username}\' logged in via the XMLRPC interface"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] Full audit trail enabled"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] User %{USERNAME:username} starts a new scan \(%{NESSUS_SCANID:scanid}\)"]
    match => [ "message","%{NESSUS_RSYSLOG:rsyslogprepend} \[%{NESSUS_DATESTAMP:timestamp}\]\[%{NUMBER:bon01}\] user %{USERNAME:username} starts a new scan. Target\(s\) : %{IPV4:ipaddress}-%{IPV4:ipaddress}, with max_hosts = %{NESSUS_MAXHOSTS:maxhosts} and max_checks = %{NESSUS_MAXCHECKS:maxchecks}"]

                    }

                    date {
            match => [ "timestamp", "EEE MMM dd HH:mm:ss yyyy" ]
            target => "@timestamp"
                    }
    }

    output {
      stdout {codec => rubydebug }
      elasticsearch {
        host => "1.1.1.1"
        port => "9200"
        protocol => "http"
        index => "nessus_scanners-%{+YYYY.MM.dd}"
      }
    #  gelf {
    #    host => "1.1.1.1"
    #  }

---

在Elasticsearch中查看您的结果是否确实存在

尝试：

---

默认情况下，Kibana正在查找索引模式为[logstash]YYYY.MM.DD的索引

该索引中有数据。我可以看到它也在Elasticseach头上。我确实将Kibana设置为[nessus scanners-]idices。我的自定义字段在Kibana中显示，但都是空的。当通过stdin从命令行导入日志文件时，Kibana会很好地填充。很抱歉指出明显的问题，但我掉进了那个陷阱，花了一些时间才弄明白。Kibana默认设置为仅显示最近几分钟的日志。因此，如果您的数据来自很久以前，除非更改默认的时间过滤器，否则您将看不到任何日志条目。

curl -XGET 'https://localhost:9200/nessus_scanners-2015.03.23/_search?pretty=true&q=*:*'