使用logstash的时候,为了更细致的切割日志,会写一些正则表达式。 使用方法 input { file { type => "billin" path => "/data/logs/product/result.log" } } filter { grok { type => "billin" pattern => "%{BILLINCENTER}" patterns_dir => "/data/logstash/patterns/my_patterns" } } output { redis { host => "192.168.50.13" data_type =>"list" key => "logstash:redis" } } 以下内容为正则表达式文件:cat my_patterns TAB META -+ WZ ([^ ]*) IPPORT %{IP}:%{POSINT}|%{META} REQUEST (?:/[A-Za-z0-9$.+!*'(),~:#%_-]*)+?[A-Za-z0-9$.+!*'(),~#%&/=:;_-]* TY (?:(?<!\)(?:"(?:\.|[^\"]+)*" #EVERYURL ((w+://)?([^.]+)(.[^/:]+)(:d*)?([^#]*))|- #EVERYURL (((w+://)?([^.]+)(.[^/:]+)?([^#]*))+)|(w+)|- #EVERYURL ((w+://)?([^.]+)(.[^/:]+)?([^#]*))+)|- EVERYURL (http://+[wd:#@%/;$()~_?+-=\.&]+)|(-) #Logformat ########nginx access log example######## #122.137.199.113"122.137.199.113"www.xxxx.com172.16.10.110172.16.12.114:8018/Jun/2013:15:51:03 +0800GET /g/getSaleCounts.do?rnd=1371541857448&showStatus=true&goodsIds=215abd2e8fa95bc8 HTTP/1.120078"http://www.xxxx.com/goods-215abd2e8fa95bc8.html""Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; NP06)""a8fdb711-a695-43bd-abdd-a224fb07350d" ############################### NGINXACCESSLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{IPPORT:upstrem_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:guid} #picture p0.xxxx.com access log . 2012.07.19 add PICLOG %{IP:remote_ip}%{SPACE}%{QS:x_forward}%{SPACE}%{HOSTNAME:server_name}%{SPACE}%{IP:server_ip}%{SPACE}%{HTTPDATE:timestamp}%{SPACE}%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER:httpversion}%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent} #iis log format 20120618 add ###########iis log example############### #2013-06-18 08:00:00 172.16.10.233 GET /js/functions.js - 80 - 117.136.34.2 Mozilla/5.0+(Linux;+U;+Android+4.1.2;+zh-CN;+LT22i+Build/6.2.A.0.400)+AppleWebKit/534.31+(KHTML,+like+Gecko)+UCBrowser/9.0.1.275+U3/0.8.0+Mobile+Safari/534.31 200 0 0 0 ################################### IISLOG %{DATE_EU:log_date} %{TIME:log_time} %{IP:server_ip} %{WORD:verb} %{URIPATH:uri_stem} %{WZ:uri_query} %{POSINT:s_port} %{WZ:cs_username} %{IP:c_ip} %{WZ:agent} %{POSINT:request} %{POSINT:substatus} %{POSINT:win32_status} %{POSINT:time_taken} #2012/07/12 add ZW w+ ###java date example # 2012-11-27 14:52:42 ############ JAVA_DATE %{DATE_EU} %{TIME} EARTHLOG [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":"%{ZW:desc}","dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}}] EAGLEUPDATE [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code},"orderId":"%{ZW:orderId}"}] EAGLELOGIN [%{JAVA_DATE:log_date}] [%{WORD:level}] [%{WORD:action}] [{"desc":%{QS:desc},"dateTime":%{ZW:dateTime},"userId":"%{ZW:userId}","code":%{ZW:code}}] #2012/10/23 add LJF (-s+-) RESINLOG %{IP:remote_ip}%{SPACE}%{NUMBER}%{SPACE}%{LJF}%{SPACE}[%{HTTPDATE:timestamp}]%{SPACE}"%{WORD:verb}%{SPACE}%{WZ:request}%{SPACE}HTTP/%{NUMBER}"%{SPACE}%{NUMBER:response}%{SPACE}%{NUMBER:bytes}%{SPACE}%{QS:uri}%{SPACE}%{QS:agent}%{SPACE}%{QS:session} #RESINLOG %{IP:ip} %{NUMBER} - - [%{HTTPDATE:time}] "%{WORD:verb} %{WZ:request} HTTP/%{NUMBER}" %{NUMVER:response} %{NUMBER:bytes} %{QS:uri} %{QS:agent} %{QS:session} #2012/11/13 add DKH ({.*}) STOREGREP ([/// - ] INFO -) DHMH ([^;|=]*) CENTERLOG %{JAVA_DATE} %{STOREGREP} BId=%{NUMBER:bid};BR=%{DHMH:br};BP=%{DKH:bp} #2012/11/20 add JAVAGREP ([/// - ]) ORDERCENTERERR %{JAVA_DATE} [ RMI TCP Connection(%{NUMER:threadid}) -%{IP:ip}] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{QS:message} ORDERCENTERRESULT %{JAVA_DATE} [ RMI TCP Connection(%{NUMER:threadid}) -%{IP:ip}] %{JAVAGREP} %{WORD:level}%{SPACE}%{WZ} - %{DKH:message} #2012/11/27 add #####log example####### #2013-06-18 15:28:12 INFO :{message:媒体传递的参数{"uid":["0"],"cid":["A100054947||0000"],"url":["http://www.xxxx.com/?from=lianmeng-weiyi"],"src":["weiyi"]}} # PARTNER %{JAVA_DATE:timestamp} %{WORD:level} :%{DKH:message} #2012/11/28 add PARTNERAPI %{JAVA_DATE:timestamp} %{WZ:level} :%{DKH:message} #2013/06/18 add #pattern all in the '[adskfjl }{]' FKH ([^;]*) #######aether.log##### #[2013-06-18 15:27:29] [INFO] [com.tuan.web.controller.IndexController] [{message:setHotStore#hot store size:5}] AETHERLOG [%{JAVA_DATE:timestamp}] [%{WZ:level}] [%{WZ:method}] %{FKH:message} USERNAME [a-zA-Z0-9._-]+ USER %{USERNAME} INT (?:[+-]?(?:[0-9]+)) BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:.[0-9]+)?)|(?:.[0-9]+))) NUMBER (?:%{BASE10NUM}) BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)) BASE16FLOAT (?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:.[0-9A-Fa-f]*)?)|(?:.[0-9A-Fa-f]+))) POSINT (?:[1-9][0-9]*) NONNEGINT (?:[0-9]+) WORD w+ NOTSPACE S+ SPACE s* DATA .*? GREEDYDATA .* #QUOTEDSTRING (?:(?<!\)(?:"(?:\.|[^\"])*"|(?:'(?:\.|[^\'])*')|(?:`(?:\.|[^\`])*`))) QUOTEDSTRING (?>(?<!\)(?>"(?>\.|[^\"]+)+"|""|(?>'(?>\.|[^\']+)+')|''|(?>`(?>\.|[^\`]+)+`)|``)) UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} # Networking MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) CISCOMAC (?:(?:[A-Fa-f0-9]{4}.){2}[A-Fa-f0-9]{4}) WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)? IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9]) IP (?:%{IPV6}|%{IPV4}) HOSTNAME (?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(.?|) HOST %{HOSTNAME} IPORHOST (?:%{HOSTNAME}|%{IP}) HOSTPORT (?:%{IPORHOST=~/./}:%{POSINT}) # paths PATH (?:%{UNIXPATH}|%{WINPATH}) UNIXPATH (?>/(?>[w_%!$@:.,-]+|\.)*)+ #UNIXPATH (?<![w/])(?:/[^/s?*]*)+ TTY (?:/dev/(pts|tty([pq])?)(w+)?/?(?:[0-9]+)) WINPATH (?>[A-Za-z]+:|\)(?:\[^\?*]*)+ URIPROTO [A-Za-z]+(+[A-Za-z+]+)? URIHOST %{IPORHOST}(?::%{POSINT:port})? # uripath comes loosely from RFC1738, but mostly from what Firefox # doesn't turn into %XX URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_-]*)+ #URIPARAM ?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? URIPARAM ?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]* URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? # Months: January, Feb, 3, 03, 12, December MONTH (?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?) MONTHNUM (?:0?[1-9]|1[0-2]) MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) # Days: Monday, Tue, Thu, etc... DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) # Years? YEAR (?>dd){1,2} # Time: HH:MM:SS #TIME d{2}:d{2}(?::d{2}(?:.d+)?)? # I'm still on the fence about using grok to perform the time match, # since it's probably slower. # TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:.%{POSINT})?)? HOUR (?:2[0123]|[01]?[0-9]) MINUTE (?:[0-5][0-9]) # '60' is a leap second in most time standards and thus is valid. SECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?) TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) ISO8601_SECOND (?:%{SECOND}|60) TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? DATE %{DATE_US}|%{DATE_EU} DATESTAMP %{DATE}[- ]%{TIME} TZ (?:[PMCE][SD]T|UTC) DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} # Syslog Dates: Month Day HH:MM:SS SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} PROG (?:[w._/%-]+) SYSLOGPROG %{PROG:program}(?:[%{POSINT:pid}])? SYSLOGHOST %{IPORHOST} SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} # Shortcuts QS %{QUOTEDSTRING} # Log formats SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} [%{HTTPDATE:timestamp}] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} # Log Levels LOGLEVEL ([A-a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)