Yii2 logs to es via logstash

Elastic Stack Logstash
1 
`I have been trying to index the logs to elasticsearch and making custom grok patterns. I have created some pattern but unable to complete the rest to match it. I would like to know whether i am dong right and if there is an easy way to do it. I have attached the sample log file if someone could take a look.
`My grok pattern ->`

%{DATESTAMP:timestamp} \[%{IP:client}]\[%{DATA:symbol}]\[%{DATA:hash}]\[%{LOGLEVEL:level}]\[%{DATA:application}:%{DATA:error_code}] %{DATA:logsource}:%{SPACE}%{DATA:logpage}:

How do i achieve this.
log file:

2019-07-11 11:03:47 [::1][\*][phrtrtus350j7ssgs9gbhc4m2h][error][yii\web\HttpException:404] yii\base\InvalidRouteException: Unable to resolve the request "applications/rest". in /var/www/html/drop_down_data/vendor/yiisoft/yii2/base/Module.php:537
Stack trace:
#0 /var/www/html/drop_down_data/vendor/yiisoft/yii2/web/Application.php(103): yii\base\Module->runAction('application-exp...', Array)
#1 /var/www/html/drop_down_data/vendor/yiisoft/yii2/base/Application.php(386): yii\web\Application->handleRequest(Object(yii\web\Request))
#2 /var/www/html/csv_func/drop_down_data/web/index.php(12): yii\base\Application->run()
#3 {main}

Next yii\web\NotFoundHttpException: Page not found. in /var/www/html/drop_down_data/vendor/yiisoft/yii2/web/Application.php:115
Stack trace:
#0 /var/www/html/drop_down_data/vendor/yiisoft/yii2/base/Application.php(386): yii\web\Application->handleRequest(Object(yii\web\Request))
#1 /var/www/html/drop_down_data/web/index.php(12): yii\base\Application->run()
#2 {main}
2019-07-11 11:03:46 [::1][-][phrtrtus350j7ssgs9gbhc4m2h][info][application] $_GET = []

$_POST = []

$_FILES = []

$_COOKIE = [
    'PHPSESSID' => 'phrtrtus350j7ssgs9gbhc4m2h'
    '_csrf' => '6ba67ab14ae5a6f0dd327c8bf0cb93530cc922dce8a247a34874d7dde1db1dada:2:{i:0;s:5:\"_csrf\";i:1;s:32:\"xKpaARLJeqwUwBBTKicCja1abpJWXtZr\";}'
]

$_SESSION = [
    '__flash' => []
    '__captcha/site/captcha' => 'taheeiz'
    '__captcha/site/captchacount' => 1
]

$_SERVER = [
    'HTTP_HOST' => 'localhost'
    'HTTP_CONNECTION' => 'keep-alive'
    'HTTP_UPGRADE_INSECURE_REQUESTS' => '1'
    'HTTP_USER_AGENT' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36'
    'HTTP_ACCEPT' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
    'HTTP_ACCEPT_ENCODING' => 'gzip, deflate, br'
    'HTTP_ACCEPT_LANGUAGE' => 'en-GB,en-US;q=0.9,en;q=0.8'
    'HTTP_COOKIE' => 'PHPSESSID=phrtrtus350j7ssgs9gbhc4m2h; _csrf=6ba67ab14ae5a6f0dd327c8bf0cb93530cc922dce8a247a34874d7dde1db1dada%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22xKpaARLJeqwUwBBTKicCja1abpJWXtZr%22%3B%7D'
    'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
    'SERVER_SIGNATURE' => '<address>Apache/2.4.29 (Ubuntu) Server at localhost Port 80</address>
'
    'SERVER_SOFTWARE' => 'Apache/2.4.29 (Ubuntu)'
    'SERVER_NAME' => 'localhost'
    'SERVER_ADDR' => '::1'
    'SERVER_PORT' => '80'
    'REMOTE_ADDR' => '::1'
    'DOCUMENT_ROOT' => '/var/www/html'
    'REQUEST_SCHEME' => 'http'
    'CONTEXT_PREFIX' => ''
    'CONTEXT_DOCUMENT_ROOT' => '/var/www/html'
    'SERVER_ADMIN' => 'webmaster@localhost'
    'SCRIPT_FILENAME' => '/var/www/html/drop_down_data/web/index.php'
    'REMOTE_PORT' => '49734'
    'GATEWAY_INTERFACE' => 'CGI/1.1'
    'SERVER_PROTOCOL' => 'HTTP/1.1'
    'REQUEST_METHOD' => 'GET'
    'QUERY_STRING' => ''
    'REQUEST_URI' => '/drop_down_data/web/index.php/application-export/generate'
    'SCRIPT_NAME' => '/drop_down_data/web/index.php'
    'PATH_INFO' => '/applications/rest'
    'PATH_TRANSLATED' => '/var/www/html/appplications/rest'
    'PHP_SELF' => '/drop_down_data/web/index.php/applications/rest'
    'REQUEST_TIME_FLOAT' => 1562843026.859
    'REQUEST_TIME' => 1562843026
]
2019-07-11 11:03:56 [::1][-][phrtrtus350j7ssgs9gbhc4m2h][info][application] $_GET = []

$_POST = []

$_FILES = []

$_COOKIE = [
    'PHPSESSID' => 'phrtrtus350j7ssgs9gbhc4m2h'
    '_csrf' => '6ba67ab14ae5a6f0dd327c8bf0cb93530cc922dce8a247a34874d7dde1db1dada:2:{i:0;s:5:\"_csrf\";i:1;s:32:\"xKpaARLJeqwUwBBTKicCja1abpJWXtZr\";}'
]

$_SESSION = [
    '__flash' => []
    '__captcha/site/captcha' => 'taheeiz'
    '__captcha/site/captchacount' => 1
]

$_SERVER = [
    'HTTP_HOST' => 'localhost'
    'HTTP_CONNECTION' => 'keep-alive'
    'HTTP_UPGRADE_INSECURE_REQUESTS' => '1'
    'HTTP_USER_AGENT' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36'
    'HTTP_ACCEPT' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'
    'HTTP_ACCEPT_ENCODING' => 'gzip, deflate, br'
    'HTTP_ACCEPT_LANGUAGE' => 'en-GB,en-US;q=0.9,en;q=0.8'
    'HTTP_COOKIE' => 'PHPSESSID=phrtrtus350j7ssgs9gbhc4m2h; _csrf=6ba67ab14ae5a6f0dd327c8bf0cb93530cc922dce8a247a34874d7dde1db1dada%3A2%3A%7Bi%3A0%3Bs%3A5%3A%22_csrf%22%3Bi%3A1%3Bs%3A32%3A%22xKpaARLJeqwUwBBTKicCja1abpJWXtZr%22%3B%7D'
    'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
    'SERVER_SIGNATURE' => '<address>Apache/2.4.29 (Ubuntu) Server at localhost Port 80</address>
'
    'SERVER_SOFTWARE' => 'Apache/2.4.29 (Ubuntu)'
    'SERVER_NAME' => 'localhost'
    'SERVER_ADDR' => '::1'
    'SERVER_PORT' => '80'
    'REMOTE_ADDR' => '::1'
    'DOCUMENT_ROOT' => '/var/www/html'
    'REQUEST_SCHEME' => 'http'
    'CONTEXT_PREFIX' => ''
    'CONTEXT_DOCUMENT_ROOT' => '/var/www/html'
    'SERVER_ADMIN' => 'webmaster@localhost'
    'SCRIPT_FILENAME' => '/var/www/html/drop_down_data/web/index.php'
    'REMOTE_PORT' => '49748'
    'GATEWAY_INTERFACE' => 'CGI/1.1'
    'SERVER_PROTOCOL' => 'HTTP/1.1'
    'REQUEST_METHOD' => 'GET'
    'QUERY_STRING' => ''
    'REQUEST_URI' => '/csv_func/drop_down_data/web/index.php/applications/rest'
    'SCRIPT_NAME' => '/csv_func/drop_down_data/web/index.php'
    'PATH_INFO' => '/applications/rest'
    'PATH_TRANSLATED' => '/var/www/html/applications/rest'
    'PHP_SELF' => '/csv_func/drop_down_data/web/index.php/applications/rest'
    'REQUEST_TIME_FLOAT' => 1562843036.921
    'REQUEST_TIME' => 1562843036
]`Preformatted text`
2

Please edit your post, select the pattern and click on </> in the toolbar above the edit pane. In the preview pane on the right you will see the appearance change to be

like this

Then do the same for the log entries.

3

I have changed it. Can you take a look now?

4

OK, so that is not a DATESTAMP. It actually matches the TIMESTAMP_ISO8601 pattern.

However, the first line is structured enough that I would use dissect rather than grok, since it is much faster.

    dissect { mapping => { "message" => "%{[@metadata][timestamp]} %{+[@metadata][timestamp]} [%{client}][%{symbol}][%{hash}][%{level}][%{application}] %{[@metadata][restOfLine]}" } }
    date { match => [ "[@metadata][timestamp]", "YYYY-MM-dd HH:mm:ss" ] }

If you ingest using a multiline codec on your file input you can combine all the lines for a given time into a single event.

multiline { pattern => "^%{TIMESTAMP_ISO8601}" negate => true what => previous auto_flush_interval => 1 }

I do not know what you want to do with the stack traces, but if you want to extract the data for the other two lines it is possible...

    if [@metadata][restOfLine] =~ /^\$_GET/ {
        dissect { mapping => { "[@metadata][restOfLine]" => "$_GET = [%{[@metadata][stuff][get]}]
$_POST = [%{[@metadata][stuff][post]}]
$_FILES = [%{[@metadata][stuff][files]}]
$_COOKIE = [%{[@metadata][stuff][cookie]}]
$_SESSION = [%{[@metadata][stuff][session]}]
$_SERVER = [%{[@metadata][stuff][server]}]%{}" } }
    }
    ruby {
        code => '
            s = event.get("[@metadata][stuff]")
            if s
                s.each { |k, v|
                    matches = v.scan(/    \'(\w+)\' => \'(.*)\'
/)
                    matches.each { |m|
                        event.set("[someField][#{k}][#{m[0]}]", m[1])
                    }
                }
            end
        '
    }
}

will get most of it. It misses a few, but you could tune the regexp with some alternation to pick up the rest. That will result in your events containing

  "someField" => {
    "cookie" => {
        "PHPSESSID" => "phrtrtus350j7ssgs9gbhc4m2h"
    },
    "server" => {
                              "PHP_SELF" => "/drop_down_data/web/index.php/applications/rest",
                           "HTTP_ACCEPT" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                           "REMOTE_ADDR" => "::1",
                     "GATEWAY_INTERFACE" => "CGI/1.1",
        "HTTP_UPGRADE_INSECURE_REQUESTS" => "1",
                             "HTTP_HOST" => "localhost",
                           "REQUEST_URI" => "/drop_down_data/web/index.php/application-export/generate",
                                  "PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
                 "CONTEXT_DOCUMENT_ROOT" => "/var/www/html",
                             "PATH_INFO" => "/applications/rest",
                        "REQUEST_SCHEME" => "http",
                  "HTTP_ACCEPT_LANGUAGE" => "en-GB,en-US;q=0.9,en;q=0.8",
                       "SERVER_SOFTWARE" => "Apache/2.4.29 (Ubuntu)",
                        "REQUEST_METHOD" => "GET",
                           "SERVER_ADDR" => "::1",
                       "SCRIPT_FILENAME" => "/var/www/html/drop_down_data/web/index.php"
    }
}
5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.