1.
filter {
multiline {
pattern => "^s+%{TIMESTAMP_ISO8601}"
negate=>true
what=>"previous"
}
mutate {
add_field => [ "[@metadata][zabbix_key]" , "logstash-api-access" ]
add_field => [ "[@metadata][zabbix_host]" , "dr-mysql01" ]
add_field =>["messager","%{type}-%{message}"]
remove_field =>["message"]
}
}
filter {
grok {
match => [ "messager","(?m)s*%{TIMESTAMP_ISO8601:time}s+(?<Level>(S+)).*"]
}
date {
match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]
}
}
[elk@yyjk conf]$ cat testtomcat.txt
2016-10-20 00:49:42,401 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:49:47,585 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:49:51,736 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:00,127 ERROR org.quartz.core.ErrorLogger - An error occurred while scanning for the next triggers to fire.
org.quartz.JobPersistenceException: Couldn't acquire next trigger: Duplicate entry 'ReportControlScheduler-auto1476870773575' for key 'PRIMARY'
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTrigger(JobStoreSupport.java:2848) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport$40.execute(JobStoreSupport.java:2759) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport$40.execute(JobStoreSupport.java:2757) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.executeInNonManagedTXLock(JobStoreSupport.java:3787) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTriggers(JobStoreSupport.java:2756) ~[quartz-2.2.1.jar:na]
at org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:272) ~[quartz-2.2.1.jar:na]
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry 'ReportControlScheduler-auto1476870773575' for key 'PRIMARY'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.8.0_20]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[na:1.8.0_20]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.8.0_20]
at java.lang.reflect.Constructor.newInstance(Constructor.java:408) ~[na:1.8.0_20]
at com.mysql.jdbc.Util.handleNewInstance(Util.java:409) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.Util.getInstance(Util.java:384) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1039) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4232) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4164) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2615) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2776) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2838) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2082) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2334) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2262) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2246) ~[mysql-connector-java-5.1.30.jar:na]
at com.mchange.v2.c3p0.impl.NewProxyPreparedStatement.executeUpdate(NewProxyPreparedStatement.java:105) ~[c3p0-0.9.1.1.jar:0.9.1.1]
at org.quartz.impl.jdbcjobstore.StdJDBCDelegate.insertFiredTrigger(StdJDBCDelegate.java:2666) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTrigger(JobStoreSupport.java:2832) ~[quartz-2.2.1.jar:na]
... 5 common frames omitted
2016-10-20 00:50:05,636 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:17,820 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:18,509 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:26,764 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:32,720 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:35,065 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:43,857 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:45,530 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:50:58,803 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:51:04,211 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:51:11,869 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
2016-10-20 00:51:12,412 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true
[elk@yyjk conf]$
[elk@yyjk conf]$ logstash -f testtomcat.conf
Settings: Default pipeline workers: 2
Defaulting pipeline worker threads to 1 because there are some filters that might not work with multiple worker threads {:count_was=>2, :filters=>["multiline"], :level=>:warn}
Pipeline main started
{
"message" => "2016-10-20 00:49:42,401 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.038Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:49:47,585 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.927Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:49:51,736 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.927Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:00,127 ERROR org.quartz.core.ErrorLogger - An error occurred while scanning for the next triggers to fire.
org.quartz.JobPersistenceException: Couldn't acquire next trigger: Duplicate entry 'ReportControlScheduler-auto1476870773575' for key 'PRIMARY'
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTrigger(JobStoreSupport.java:2848) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport$40.execute(JobStoreSupport.java:2759) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport$40.execute(JobStoreSupport.java:2757) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.executeInNonManagedTXLock(JobStoreSupport.java:3787) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTriggers(JobStoreSupport.java:2756) ~[quartz-2.2.1.jar:na]
at org.quartz.core.QuartzSchedulerThread.run(QuartzSchedulerThread.java:272) ~[quartz-2.2.1.jar:na]
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry 'ReportControlScheduler-auto1476870773575' for key 'PRIMARY'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.8.0_20]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[na:1.8.0_20]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[na:1.8.0_20]
at java.lang.reflect.Constructor.newInstance(Constructor.java:408) ~[na:1.8.0_20]
at com.mysql.jdbc.Util.handleNewInstance(Util.java:409) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.Util.getInstance(Util.java:384) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1039) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4232) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4164) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2615) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2776) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2838) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2082) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2334) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2262) ~[mysql-connector-java-5.1.30.jar:na]
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2246) ~[mysql-connector-java-5.1.30.jar:na]
at com.mchange.v2.c3p0.impl.NewProxyPreparedStatement.executeUpdate(NewProxyPreparedStatement.java:105) ~[c3p0-0.9.1.1.jar:0.9.1.1]
at org.quartz.impl.jdbcjobstore.StdJDBCDelegate.insertFiredTrigger(StdJDBCDelegate.java:2666) ~[quartz-2.2.1.jar:na]
at org.quartz.impl.jdbcjobstore.JobStoreSupport.acquireNextTrigger(JobStoreSupport.java:2832) ~[quartz-2.2.1.jar:na]
... 5 common frames omitted",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.928Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat",
"tags" => [
[0] "multiline"
]
}
{
"message" => "2016-10-20 00:50:05,636 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.991Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:17,820 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.991Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:18,509 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.992Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:26,764 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.992Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:32,720 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.993Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:35,065 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.993Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:43,857 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.993Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:45,530 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.994Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:50:58,803 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.994Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:51:04,211 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.995Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:51:11,869 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.995Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}
{
"message" => "2016-10-20 00:51:12,412 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
"@version" => "1",
"@timestamp" => "2018-09-10T09:11:59.996Z",
"path" => "/home/elk/conf/testtomcat.txt",
"host" => "yyjk",
"type" => "tomcat"
}