Patterns :
Access log Apache
1 #Gestion des logs access apache dans syslog
2 grok {
3 match => { "message" => "%{SYSLOG5424PRI}%{SYSLOGBASE2} %{COMBINEDAPACHELOG}" }
4 add_tag => [ "apache" , "accesslogs" ]
5 }
iptables
1 grok {
2 patterns_dir => [ "/etc/logstash/patterns" ]
3 match => { "message" => "%{IPTABLES}" }
4 add_tag => [ "iptables" , "iptables-drop" ]
5 }
Fichier de pattern :
1 DHCPD (( %{SYSLOGTIMESTAMP:timestamp} )\ s ( %{HOSTNAME:hostname} )\ s dhcpd \ S + \ s ( %{WORD:dhcp_action} ) ?. [ for | on ] ( %{IPV4:dhcp_client_ip} ) ?. [ from | to ] ( %{COMMONMAC:dhcp_client_mac} ) ?. via ( %{USERNAME:interface} ))
2 #IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s(%{HOSTNAME:nf_host})\s kernel\S+\s(%{WORD:nf_action})?. IN=(%{USERNAME:nf_in_interface})?.OUT=(%{USERNAME:nf_out_interface})?. MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.SRC=(%{IPV4:nf_src_ip}). DST=(%{IPV4:nf_dst_ip}).PROTO=(%{WORD:nf_protocol}).? SPT=(%{INT:nf_src_port}?.DPT=%{INT:nf_dst_port}?. ))
3 #Gestion de IPv4 et v6
4 IPTABLES (( %{SYSLOGTIMESTAMP:nf_timestamp} )\ s ( %{HOSTNAME:nf_host} )\ s kernel \ S + \ s ( %{WORD:nf_action} ) ?. IN = ( %{USERNAME:nf_in_interface} ) ?. OUT = ( %{USERNAME:nf_out_interface} ) ?. MAC = ( %{COMMONMAC:nf_dst_mac} ):( %{COMMONMAC:nf_src_mac} ) ?. SRC = ( %{IP:nf_src_ip} ) . DST = ( %{IP:nf_dst_ip} ) . PROTO = ( %{WORD:nf_protocol} ) . ? SPT = ( %{INT:nf_src_port} ?. DPT = %{INT:nf_dst_port} ?. ))
5 DNS (( %{MONTHDAY:day} ) - ( %{MONTH:month} ) - ( %{YEAR:year} ) ( %{TIME:timestamp} ) client ( %{IPV4:dns_client_ip} ) #(%{NONNEGINT:dns_uuid})?.query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?. (%{IPV4:dns_server}))
6 PGSQL (( %{SYSLOGTIMESTAMP:pgsql_timestamp} ) ( %{HOSTNAME:pgsql_hostname} ) ?. * SAST > ( %{WORD:pgsql_severity} ): ( %{GREEDYDATA:pgsql_message} ))
7 SQUID (( %{DATA:squid_timestamp_unix} ) ( %{INT:squid_time_elapsed_ms} ) ( %{IPV4:squid_client_ip} ) ( %{DATA:squid_action} ) / ( %{INT:squid_reply_code} ) ( %{INT:squid_request_size} ) ( %{WORD:squid_method} ) ( %{URIPROTO:squid_request_protocol} ) :/ / ( %{DATA:squid_request_hostname} ) - ( %{WORD:squid_heirarchy} ) / ( %{IPV4:squid_dest_ip} ) ( %{WORD:squid_content_type} ) / ( %{WORD:squid_content_object} ))
8 RADIUSINFO (( %{SYSLOGTIMESTAMP:radius_timestamp} ) ( %{YEAR} ) : ( %{WORD:radius_severity} ): ( %{GREEDYDATA:radius_messsage} ))
9 RADIUSERROR (( %{SYSLOGTIMESTAMP:radius_timestamp} ) ( %{YEAR} ) : ( %{WORD:radius_severity} ): ( %{DATA:radius_module} ): ( %{GREEDYDATA:radius_messsage} ))
10 RADIUSAUTHOKMAC (( %{SYSLOGTIMESTAMP:radius_timestamp} ) ( %{YEAR} ) : ( %{WORD:radius_severity} ): ( %{GREEDYDATA:radius_login_status} ) \ [ ( %{USERNAME:radius_user} ) / ( %{USERNAME:radius_password} )\ ] ( from client ( %{GREEDYDATA:radius_from_client} ) port ( %{INT:radius_nas_port} ) ( %{WORD} ) ( %{GREEDYDATA:radius_mac} )))
11 RADIUSAUTHOK (( %{SYSLOGTIMESTAMP:radius_timestamp} ) ( %{YEAR} ) : ( %{WORD:radius_severity} ): ( %{GREEDYDATA:radius_login_status} ) \ [ ( %{USERNAME:radius_user} ) / ( %{USERNAME:radius_password} )\ ] ( from client ( %{GREEDYDATA:radius_from_client} ) port ( %{INT:radius_nas_port} ))
12 RADIUSAUTHFAIL (( %{SYSLOGTIMESTAMP:radius_timestamp} ) ( %{YEAR} ) : ( %{WORD:radius_severity} ): ( %{GREEDYDATA:radius_login_status} ): \ [ ( %{USERNAME:radius_user} ) / ( %{USERNAME:radius_password} )\ ] ( from client ( %{GREEDYDATA:radius_from_client} ) port ( %{INT:radius_nas_port} ))
fail2ban
1 grok {
2 match => { "message" => "(%{SYSLOGTIMESTAMP:f2b_timestamp}) \s (%{HOSTNAME:f2b_host}) \s fail2ban.action\S+ \s (%{WORD:f2b_level}) \s [(%{GREEDYDATA:f2b_jail}) \s ] \s (%{WORD:f2b_action}) \s *(%{IPORHOST:src_ip})" }
3 add_tag => [ "fail2ban" ]
4 }
DenyALL rWeb
Trafic logs
1 grok {
2 keep_empty_captures => true
3 match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3}) \s +(?<first_timestamp_dd>\d{1,2}) \s +(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2}) \s +(?<device>\S+) \s +(?<program>\S+) \s +(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2}) \s +(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2}).(?<timestamp_SSSSS>\d+) \s +(?<timestamp_ZZ>[^,]),%{IP:local_ip},(?<hostname>[^,]+),%{IP:remote_ip},(?<x_forwarded_for>( " ([^ " ]) " )|([^,])),(?<via>[^,] ),(?<user>[^,]),(?<user_agent>( " +[^ " ]+ " +|[^,])),(?<SSL>\d),(?<SSL_version>[^,]),(?<SSL_client_DN>[^,] ),(?<SSL_certificate_start>[^,]),(?<SSL_certificate_end>[^,] ),(?<HTTP_method>[^,]+),(?<URL>[^,]+),(?<URL_options>.),(?<HTTP_version>HTTP[^,]+),%{NUMBER:HTTP_response_code:int},%{NUMBER:HTTP_response_time:int},%{NUMBER:bytes_received:int},%{NUMBER:bytes_sent:int},(?<referrer>[^,] ),(?<cache>[^,]),(?<gzip_ratio>[^,] ),(?<applicationID>[^,]+),(?<source_port>\d+),(?<application>[^,]+),(?<unique_id>[^,]+),(?<cookie>[^,])(,?)(?<posted_data>. )" }
4 # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
5 add_field => [ "timezone" , "+0100" ]
6 add_field => [ "log_type" , "traffic" ]
7 add_tag => [ "rweb_logs" , "rweb_traffic_logs" ]
8 }
Alert logs
1 grok {
2 keep_empty_captures => true
3 match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3}) \s +(?<first_timestamp_dd>\d{1,2}) \s +(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2}) \s +(?<device>\S+) \s +\S+ \s +(?<log_id>\d+) \s +(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2}) \s +(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2}).(?<timestamp_SSSSS>\d+) \s +%{IP:source_ip} \s +%{IP:remote_ip} \s +\S+ \s +(?<alert_id>\d+.\d+.\d+.\d+) \s +(?<applicationID>\S+) \s +(?<unique_id>\S+) \s +(?<sec_bl_sl_id>(([0-9]{5}|.{0})-[0-9]{0,3} \s )+) \s (?<sec_attack_id>([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12}|-)) \s + ' (?<description>[^']) '\s + ' (?<attack_family>[^']+) ' " }
4
5 # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
6 add_field => [ "timezone" , "+0100" ]
7 add_field => [ "log_type" , "alert" ]
8 # Duplicate field for get Application Name after
9 add_field => [ "application" , "%{applicationID}" ]
10 add_tag => [ "rweb_logs" , "rweb_alert_logs" ]
11 }
Du fait que les Alert logs ne fournissent pas le nom d'application mais que l'UID, on duplique l'UID dans un nouveau champ (application) et ensuite on modifie de manière statique (ou scripté) via un mutate .
1 mutate {
2 # remove_field => [ "message" ]
3 } alter {
4 condrewrite => [
5 "user" , "" , "-" ,
6 "bytes_received" , "" , "-" ,
7 "referrer" , "" , "-" ,
8 "application" , "56b7df6c-e9e6-454d-9ab5-45d357aaf5e7" , "APP_NAME" ,
9 "timestamp_MM" , "01" , "Jan" ,
10 "timestamp_MM" , "02" , "Feb" ,
11 "timestamp_MM" , "03" , "Mar" ,
12 "timestamp_MM" , "04" , "Apr" ,
13 "timestamp_MM" , "05" , "May" ,
14 "timestamp_MM" , "06" , "Jun" ,
15 "timestamp_MM" , "07" , "Jul" ,
16 "timestamp_MM" , "08" , "Aug" ,
17 "timestamp_MM" , "09" , "Sep" ,
18 "timestamp_MM" , "10" , "Oct" ,
19 "timestamp_MM" , "11" , "Nov" ,
20 "timestamp_MM" , "12" , "Dec" ]
21 }
22 }
Aide au debug
Aide/debug de pattern grok