ELK - Logstash exemples de patterns grok

1 #Gestion des logs access apache dans syslog
2 grok {
3   match => { "message" => "%{SYSLOG5424PRI}%{SYSLOGBASE2} %{COMBINEDAPACHELOG}" }
4   add_tag => [ "apache", "accesslogs" ]
5 }

1 grok {
2   patterns_dir => ["/etc/logstash/patterns"]
3   match => { "message" => "%{IPTABLES}"}
4   add_tag => [ "iptables", "iptables-drop"]
5 }

 1 DHCPD ((%{SYSLOGTIMESTAMP:timestamp})\s(%{HOSTNAME:hostname})\sdhcpd\S+\s(%{WORD:dhcp_action})?.[for|on] (%{IPV4:dhcp_client_ip})?.[from|to] (%{COMMONMAC:dhcp_client_mac})?.via (%{USERNAME:interface}))
 2 #IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s(%{HOSTNAME:nf_host})\skernel\S+\s(%{WORD:nf_action})?.IN=(%{USERNAME:nf_in_interface})?.OUT=(%{USERNAME:nf_out_interface})?.MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.SRC=(%{IPV4:nf_src_ip}).DST=(%{IPV4:nf_dst_ip}).PROTO=(%{WORD:nf_protocol}).?SPT=(%{INT:nf_src_port}?.DPT=%{INT:nf_dst_port}?.))
 3 #Gestion de IPv4 et v6
 4 IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s(%{HOSTNAME:nf_host})\skernel\S+\s(%{WORD:nf_action})?.IN=(%{USERNAME:nf_in_interface})?.OUT=(%{USERNAME:nf_out_interface})?.MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.SRC=(%{IP:nf_src_ip}).DST=(%{IP:nf_dst_ip}).PROTO=(%{WORD:nf_protocol}).?SPT=(%{INT:nf_src_port}?.DPT=%{INT:nf_dst_port}?.))
 5 DNS ((%{MONTHDAY:day})-(%{MONTH:month})-(%{YEAR:year}) (%{TIME:timestamp}) client (%{IPV4:dns_client_ip})#(%{NONNEGINT:dns_uuid})?.query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?.(%{IPV4:dns_server}))
 6 PGSQL ((%{SYSLOGTIMESTAMP:pgsql_timestamp}) (%{HOSTNAME:pgsql_hostname})?.*SAST >(%{WORD:pgsql_severity}):  (%{GREEDYDATA:pgsql_message}))
 7 SQUID ((%{DATA:squid_timestamp_unix}) (%{INT:squid_time_elapsed_ms}) (%{IPV4:squid_client_ip}) (%{DATA:squid_action})/(%{INT:squid_reply_code}) (%{INT:squid_request_size}) (%{WORD:squid_method}) (%{URIPROTO:squid_request_protocol})://(%{DATA:squid_request_hostname}) - (%{WORD:squid_heirarchy})/(%{IPV4:squid_dest_ip}) (%{WORD:squid_content_type})/(%{WORD:squid_content_object}))
 8 RADIUSINFO ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_messsage}))
 9 RADIUSERROR ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{DATA:radius_module}): (%{GREEDYDATA:radius_messsage}))
10 RADIUSAUTHOKMAC ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] (from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}) (%{WORD}) (%{GREEDYDATA:radius_mac})))
11 RADIUSAUTHOK ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] (from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
12 RADIUSAUTHFAIL ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}): \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] (from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))

1 grok {
2   match => { "message" => "(%{SYSLOGTIMESTAMP:f2b_timestamp})\s(%{HOSTNAME:f2b_host})\sfail2ban.action\S+\s(%{WORD:f2b_level})\s[(%{GREEDYDATA:f2b_jail})\s]\s(%{WORD:f2b_action})\s*(%{IPORHOST:src_ip})" }
3   add_tag => [ "fail2ban" ]
4 }

1 grok {
2   keep_empty_captures => true
3   match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+(?<program>\S+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2}).(?<timestamp_SSSSS>\d+)\s+(?<timestamp_ZZ>[^,]),%{IP:local_ip},(?<hostname>[^,]+),%{IP:remote_ip},(?<x_forwarded_for>(&quot;([^&quot;])&quot;)|([^,])),(?<via>[^,]),(?<user>[^,]),(?<user_agent>(&quot;+[^&quot;]+&quot;+|[^,])),(?<SSL>\d),(?<SSL_version>[^,]),(?<SSL_client_DN>[^,]),(?<SSL_certificate_start>[^,]),(?<SSL_certificate_end>[^,]),(?<HTTP_method>[^,]+),(?<URL>[^,]+),(?<URL_options>.),(?<HTTP_version>HTTP[^,]+),%{NUMBER:HTTP_response_code:int},%{NUMBER:HTTP_response_time:int},%{NUMBER:bytes_received:int},%{NUMBER:bytes_sent:int},(?<referrer>[^,]),(?<cache>[^,]),(?<gzip_ratio>[^,]),(?<applicationID>[^,]+),(?<source_port>\d+),(?<application>[^,]+),(?<unique_id>[^,]+),(?<cookie>[^,])(,?)(?<posted_data>.)"}
4   # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
5   add_field => [ "timezone", "+0100" ]
6   add_field => [ "log_type", "traffic" ]
7   add_tag => [ "rweb_logs", "rweb_traffic_logs" ]
8 }

 1 grok {
 2  keep_empty_captures => true
 3  match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+\S+\s+(?<log_id>\d+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2}).(?<timestamp_SSSSS>\d+)\s+%{IP:source_ip}\s+%{IP:remote_ip}\s+\S+\s+(?<alert_id>\d+.\d+.\d+.\d+)\s+(?<applicationID>\S+)\s+(?<unique_id>\S+)\s+(?<sec_bl_sl_id>(([0-9]{5}|.{0})-[0-9]{0,3}\s)+)\s(?<sec_attack_id>([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12}|-))\s+&#39;(?<description>[^'])&#39;\s+&#39;(?<attack_family>[^']+)&#39;" }
 4 
 5  # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
 6  add_field => [ "timezone", "+0100" ]
 7  add_field => [ "log_type", "alert" ]
 8  # Duplicate field for get Application Name after
 9  add_field => [ "application", "%{applicationID}" ]
10  add_tag => [ "rweb_logs" , "rweb_alert_logs"]
11 }

 1 mutate {
 2  #  remove_field => [ "message" ]
 3  } alter {
 4  condrewrite => [
 5  "user", "", "-",
 6  "bytes_received", "", "-",
 7  "referrer", "", "-",
 8  "application", "56b7df6c-e9e6-454d-9ab5-45d357aaf5e7", "APP_NAME",
 9  "timestamp_MM", "01", "Jan",
10  "timestamp_MM", "02", "Feb",
11  "timestamp_MM", "03", "Mar",
12  "timestamp_MM", "04", "Apr",
13  "timestamp_MM", "05", "May",
14  "timestamp_MM", "06", "Jun",
15  "timestamp_MM", "07", "Jul",
16  "timestamp_MM", "08", "Aug",
17  "timestamp_MM", "09", "Sep",
18  "timestamp_MM", "10", "Oct",
19  "timestamp_MM", "11", "Nov",
20  "timestamp_MM", "12", "Dec" ]
21  }
22 }