Fatal编程技术网
  • logstash/
  • Syslog消息的多个部分logstash可以重新组装吗？

Syslog消息的多个部分logstash可以重新组装吗？

logstash

Syslog消息的多个部分logstash可以重新组装吗？,logstash,Logstash,我们正在从Cisco设备收集CISE日志。消息分多个部分到达。logstash能重新组装它们吗 "message" => "<181>Dec 13 20:41:35 sfm-ise-psn1 CISE_Passed_Authentications 0069245712 3 1 IdentityGroup=Endpoint Identity Groups:SFDC-Assets:SFDC-AccessPoints, Step=11001, Step=11017, Step=11

我们正在从Cisco设备收集CISE日志。消息分多个部分到达。logstash能重新组装它们吗

"message" => "<181>Dec 13 20:41:35 sfm-ise-psn1 CISE_Passed_Authentications 0069245712 3 1  IdentityGroup=Endpoint Identity Groups:SFDC-Assets:SFDC-AccessPoints, Step=11001, Step=11017, Step=11027, Step=15049, Step=15008, Step=15048, Step=15048, Step=15004, Step=15041, Step=15006, Step=15013, Step=24209, Step=24211, Step=22037, Step=15036, Step=15048, Step=15048, Step=15048, Step=15004, Step=15016, Step=11022, Step=11002, SelectedAuthenticationIdentityStores=Internal Endpoints, NetworkDeviceGroups=Stage#Stage#Low Impact Mode, NetworkDeviceGroups=Location#All Locations#SFDC#SFDC-Americas#SFDC-Americas-East#NY-New York NEW, NetworkDeviceGroups=Device Type#All Device Types#Switching, AuthorizationPolicyMatchedRule=SFDC Access Points, UserType=Host, CPMSessionID=000000000000000200016F9F, EndPointMACAddress=64-9E-F3-B3-5C-15, PostureAssessmentStatus=NotApplicable, EndPointMatchedProfile=Cisco-AP-Aironet-1140, DeviceRegistrationStatus=notRegistered, ISEPolicySetName=Default, AllowedProtocolMatchedRule=MAB,",
"message" => "<181>Dec 13 20:41:35 sfm-ise-psn1 CISE_Passed_Authentications 0069245712 3 2  IdentitySelectionMatchedRule=Default, HostIdentityGroup=Endpoint Identity Groups:SFDC-Assets:SFDC-AccessPoints, Stage=Stage#Stage#Low Impact Mode, Location=Location#All Locations#SFDC#SFDC-Americas#SFDC-Americas-East#NY-New York NEW, Device Type=Device Type#All Device Types#Switching, PostureStatus=Unknown, Response={UserName=64:9E:F3:B3:5C:15; User-Name=64-9E-F3-B3-5C-15; State=ReauthSession:000000000000000200016F9F; Class=CACS:000000000000000200016F9F:sfm-ise-psn1/228424214/80382355; Session-Timeout=14400; Termination-Action=RADIUS-Request; cisco-av-pair=ACS:CiscoSecure-Defined-ACL=#ACSACL#-IP-PERMIT_ALL_TRAFFIC-5165e13c; cisco-av-pair=profile-name=Cisco-AP-Aironet-1140; },",
"message" => "<181>Dec 13 20:41:35 sfm-ise-psn1 CISE_Passed_Authentications 0069245712 3 0 2015-12-13 20:41:35.925 +00:00 9534852286 5200 NOTICE Passed-Authentication: Authentication succeeded, ConfigVersionId=61, Device IP Address=10.119.68.254, DestinationIPAddress=10.1.64.13, DestinationPort=1812, UserName=64-9E-F3-B3-5C-15, Protocol=Radius, RequestLatency=7, NetworkDeviceName=Melville-3750_copy, User-Name=649ef3b35c15, NAS-IP-Address=10.119.68.254, NAS-Port=50122, Service-Type=Call Check, Framed-IP-Address=10.119.68.3, Framed-MTU=1500, Called-Station-ID=28-94-0F-34-AC-16, Calling-Station-ID=64-9E-F3-B3-5C-15, NAS-Port-Type=Ethernet, NAS-Port-Id=GigabitEthernet1/0/22, EAP-Key-Name=, cisco-av-pair=audit-session-id=000000000000000200016F9F, OriginalUserName=649ef3b35c15, AcsSessionID=sfm-ise-psn1/228424214/80382355, AuthenticationIdentityStore=Internal Endpoints, AuthenticationMethod=Lookup, SelectedAccessService=Default Network Access, SelectedAuthorizationProfiles=PermitAccess-SFDC-AP, UseCase=Host Lookup,",

“消息”=>“Dec 13 20:41:35 sfm-ise-psn1 CISE通过认证0069245712 3 1身份组=端点身份组：SFDC资产：SFDC访问点，步骤=11001，步骤=11017，步骤=11027，步骤=15049，步骤=15008，步骤=15048，步骤=15048，步骤=15004，步骤=15041，步骤=15006，步骤=15013，步骤=24209，步骤=24211，步骤=22037，步骤=15036，步骤=15048，步骤=15048，Step=15048，Step=15004，Step=15016，Step=11022，Step=11002，SelectedAuthenticationIdentityStores=Internal Endpoints，NetworkDeviceGroups=Stage#Stage#Stage#Low Impact Mode，NetworkDeviceGroups=Location#All Location#SFDC#SFDC Americas#SFDC Americas East#NY New York New，NetworkDeviceGroups=设备类型#切换，AuthorizationPolicyMatchedRule=SFDC访问点，UserType=Host，CPMSessionID=000000000000000 200016F9F，EndPointMACAddress=64-9E-F3-B3-5C-15，PositureReassessmentStatus=NotApplieble，EndPointMatchedProfile=Cisco-AP-Aironet-1140，DeviceRegistrationStatus=notRegistered，ISEPolicySetName=Default，AllowedProtocolMatchedRule=MAB，“，
“message”=>“Dec 13 20:41:35 sfm-ise-psn1 CISE通过认证0069245712 Identity SelectionMatchedRule=默认，HostIdentity Group=端点标识组：SFDC资产：SFDC访问点，阶段=阶段#阶段#低影响模式，位置=位置#所有位置#SFDC#SFDC美洲#SFDC美洲东部#纽约#，设备类型=设备类型#所有设备类型#切换，姿势复位=未知，响应={用户名=64:9E:F3:B3:5C:15；用户名=64-9E-F3-B3-5C-15；状态=ReauthSession:000000000000000 200016F9F；类别=CACS:000000000000000 200016F9F:sfm-ise-psn1/228424214/80382355；会话超时=14400；终止操作=RADIUS请求；cisco av对=ACS:cisco安全定义的ACL=#ACSACL#-IP-PERMIT\U ALL\u流量-5165e13c；cisco av对=profile=-Aironet-1140；}，“，
“消息”=>”12月13日20:41:35 sfm-ise-psn1 CISE通过认证0069245712 3 0 2015-12-13 20:41:35.925+00:00 9534852286 5200通知通过认证：认证成功，ConfigVersionId=61，设备IP地址=10.119.68.254，DestinationPaddress=10.1.64.13，DestinationPort=1812，用户名=64-9E-F3-B3-5C-15，协议=Radius，请求延迟=7，网络workDeviceName=Melville-3750_copy，用户名=649ef3b35c15，NAS IP地址=10.119.68.254，NAS端口=50122，服务类型=呼叫检查，框架IP地址=10.119.68.3，框架MTU=1500，被叫站ID=28-94-0F-34-AC-16，主叫站ID=64-9E-F3-B3-5C-15，NAS端口类型=以太网，NAS端口ID=GigabitEthernet1/0/22，EAP密钥名=，cisco av对=审计se会话id=000000000000000 200016F9F，OriginalUserName=649ef3b35c15，AcsSessionID=sfm-ise-psn1/228424214/80382355，AuthenticationIdentityStore=内部端点，AuthenticationMethod=查找，SelectedAccessService=默认网络访问，SelectedAuthorizationProfiles=许可访问SFDC AP，UseCase=主机查找，“，
在看了聚合插件之后，我觉得我离它越来越近了。这个消息还没有组装成一个条目

if [message] =~ /^<181>/ {
grok {
  match => { "message" => "%{SYSLOG5424PRI:pri}%{CISCOTIMESTAMP:time} %{IPORHOST:hostname} %{WORD:type} %{INT:taskid} %{INT:duration:int} %{INT:order:int} "}
}
    if [order] == "0" {
         aggregate {
            task_id => "%{taskid}"
            code => "map['sql_duration'] ||= 0 ; map['sql_duration'] += event['duration']"
        }
    }
    if [order] == "%{duration}" {
        aggregate {
            task_id => "%{taskid}"
            code => "event['sql_duration'] = map['sql_duration']"
            end_of_task => true
            timeout => 120
        }
    }
kv {
  type => syslog
  add_field => { "log_type" => "CISE" }
}
  }

如果[消息]=~/^/{
格罗克{
match=>{“message”=>“%{SYSLOG5424PRI:pri}%{CISCOTIMESTAMP:time}%{IPORHOST:hostname}%{WORD:type}%{INT:taskid}%{INT:duration:INT}%{INT:order:INT}
}
如果[订单]=“0”{
聚合{
任务id=>“%{taskid}”
代码=>“映射['sql_duration']| |=0；映射['sql_duration']+=event['duration']”
}
}
如果[顺序]=“%{duration}”{
聚合{
任务id=>“%{taskid}”
代码=>“事件['sql\U duration']=map['sql\U duration']”
结束任务的任务=>true
超时=>120
}
}
千伏{
类型=>syslog
添加\u字段=>{“日志类型”=>“CISE”}
}
}
因此，虽然与日志存储无关，但ISE截断日志的原因是系统日志消息的可配置大小。这可以配置多达8000字节（对于单个消息）这意味着您不必使用ISE会话ID来加入邮件。

我知道这有点旧，但下面是我如何解决它的，以防有人在ISE&Logstash中遇到同样的问题

grok {
    match => [
        "_message", "(?:\<%{POSINT:syslog_pri}\>)?%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:hostname} %{WORD:kind} %{NOTSPACE:taskid} %{INT:duration:int} %{INT:order:int} (%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{ISO8601_TIMEZONE})? %{GREEDYDATA:rest}"
    ]
}       

#If this is the first part create a map and save copy of message to it 
if [order] == 0 { 
    aggregate {
        task_id => "%{taskid}"
        code => "map['message'] ||= {} ; map['message']['0'] = event.get('_message')"
        map_action => "create"
    }
} else {
    #Keep adding messages to the array until the last part is received
    # This is to ensure messages are in the correct order
    aggregate {
        task_id => "%{taskid}"
        code => "
            map['message'][event.get('order').to_s] = event.get('_message')
            event.set('message_parts', map['message'])

            if map['message'].length == event.get('duration') then
                mytags = event.get('tags')
                mytags << 'finish'
                event.set('tags', mytags)
            end
        "
        map_action => "update"
    }
}

# If we got the last part start concatenating messages else drop the event (We have a copy in map)
if "finish" in [tags] { 
    # Ruby code to convert the array to a well formatted single string
    ruby {
        code => "
            msg = ''
            a = event.get('[message_parts]')
            num = a.length

            for i in 0..num-1
                msg +=  a[i.to_s]
            end

            # set the message to the new concatenated one
            # but make sure to remove message header (PRI, timestamp, etc..) from all except first part
            event.set('message', msg.gsub(/(?<!^)<\d{3}>\w{3} \d{2} \d{2}:\d{2}:\d{2} (.*?) (.*?) (.*?) \d+ \d+ /, ''))
        "
    }

    mutate {
        remove_field => [ "taskid", "order", "duration", "hostname", "kind", "rest" , "syslog_pri", "_message"]
    }

    grok {
        match => [
            "message", "(?:\<%{POSINT:syslog_pri}\>)?%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:hostname} %{WORD:kind} %{NOTSPACE:taskid} %{INT:duration:int} %{INT:order:int} %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{TIME} %{ISO8601_TIMEZONE} %{INT:sequence_num:int} %{INT:msg_code:int} %{DATA:msg_sev} %{DATA:msg_class}: %{DATA:msg_text}, %{GREEDYDATA:attributes}"
        ]
    }
} else {
    # Ignore individual parts
    drop{}
}

grok{
匹配=>[
“\u message”、“（？：\）？%{SYSLOGTIMESTAMP:timestamp}%{IPORHOST:hostname}%{WORD:kind}%{NOTSPACE:taskid}%{INT:duration:INT}%{INT:order:INT}%{YEAR:INT}%{MONTHNUM}-%{MONTHDAY}%{TIME}%{ISO8601\u时区}”{greedydydydata rest}”
]
}       
#如果这是第一部分，请创建地图并将消息副本保存到地图
如果[顺序]==0{
聚合{
任务id=>“%{taskid}”
代码=>“映射['message']|={}；映射['message']['0']=event.get（'u message'）”
映射_操作=>“创建”
}
}否则{
#继续向阵列添加消息，直到收到最后一部分
#这是为了确保消息的顺序正确
聚合{
任务id=>“%{taskid}”
代码=>”
映射['message'][event.get（'order'）。到事件]=event.get（'u message'））
事件集（'message_parts'，map['message']））
如果map['message'].length==event.get（'duration'），则
mytags=event.get（'tags'）
mytags）%{SYSLOGTIMESTAMP:timestamp}%{IPORHOST:hostname}%{WORD:kind}%{NOTSPACE:taskid}%{INT:duration:INT}%{INT:order:INT}%{YEAR}-%{MONTHNUM}-%{MONTHDAY}%{TIME}%{ISO8601\u时区}%{INT:sequence\u num INT}%{INT:msg代码：INT}%{DATA:msg\sev}%{DATA:msg}-{monthnay}%{MONTHDAY}%{TIME}%{ISO8601\TIMEZONE}%{TIMEZONE}%{INT:sevu时区}%{INT:msg}%{INT}%{INT:msg}%{
]
}
}否则{
#忽略单个零件
删除{}
}
使用“聚合”筛选器可能是可行的：不幸的是，这并不完全正确。尽管您可以将消息大小设置为8192个“是”，但仍然可以经常找到（特别是使用CISE\u Passed\u Authentications log）您仍然会将其分为两部分。哦，这也适用于TCP系统日志。它将在将消息传递到某个本地系统日志服务器之前拆分消息