Not able to parse completely with grok

Elastic Stack Logstash
1 
  "10/Nov/2023:12:59:05 +0530" 192.54.23.32 GET "GET /healthcheck HTTP/1.1" 178 200 453 2  "nginx/1.23.4 (health check server_192.87.23.870_Pool-1_http_ok)" 127.0.0.1:3000 200 0.001 0.002 0.000 0.002  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:05 +0530" 192.54.23.32 GET "GET /healthcheck HTTP/1.1" 200 200 453 2  "nginx/1.23.4 (health check server_192.87.23.870_Pool-1_https_ok)" 127.0.0.1:3000 200 0.001 0.001 0.000 0.001  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:05 +0530" 192.45.108.26 GET "GET /healthcheck HTTP/1.1" 179 200 453 2  "nginx/1.23.4 (health check server_192.45.108.199_Pool-1_http_ok)" 127.0.0.1:3000 200 0.002 0.001 0.000 0.001  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:05 +0530" 192.45.108.26 GET "GET /healthcheck HTTP/1.1" 201 200 453 2  "nginx/1.23.4 (health check server_192.45.108.199_Pool-1_https_ok)" 127.0.0.1:3000 200 0.001 0.001 0.000 0.001  apimumbcms.hydtimes.com
192.87.205.186 "10/Nov/2023:12:59:05 +0530" 192.87.23.107 GET "GET /socket.io/?EIO=4&transport=polling&t=Oku8ckz HTTP/1.1" 721 200 627 122 https://delhi.indiatimes.com/ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" 127.0.0.1:3000 200 0.002 0.001 0.000 0.001  mumbcms.hydtimes.com
192.87.205.186 "10/Nov/2023:12:59:05 +0530" 192.87.23.107 GET "GET /socket.io/?EIO=4&transport=polling&t=Oku8col HTTP/1.1" 721 200 630 187 https://delhi.indiatimes.com/ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" 127.0.0.1:3000 200 0.002 0.001 0.000 0.001  mumbcms.hydtimes.com
192.87.80.63 "10/Nov/2023:12:59:06 +0530" 192.87.23.121 POST "POST /api/cache/purge HTTP/1.1" 860 200 511 51  "Apache-HttpClient/4.5.2 (Java/1.8.0-262)" 127.0.0.1:3000 200 5.305 5.305 0.000 5.305  apimumbcms.hydtimes.com {\"msid\":\"105116742\",\"keywords\":\"Dhanteras 2023%2CDhanteras 2023 Wishes%2CDhanteras 2023 Quotes%2CDhanteras 2023 Messages%2CDhanteras Images%2Cdhanteras festival%2CDhanteras Puja Upay%2CDhanteras Worship Method%2C\",\"hostId\":\"155\",\"mstype\":\"8\",\"title\":\"धनत्रयोदशीच्या दिवशी या शुभ वस्तू खरेदी करा आणि घरात सौभाग्य आणा\",\"type\":\"PHOTOGALLERYSLIDESHOWSECTION\",\"hisactive\":\"0\",\"operation\""mssubtype\":\"1\",\"parentId\":\"87872838\"}
192.87.82.84 "10/Nov/2023:12:59:08 +0530" 192.87.23.115 POST "POST /api/cache/purge HTTP/1.1" 545 200 511 51  "Apache-HttpClient/4.5.2 (Java/1.8.0-262)" 127.0.0.1:3000 200 0.055 0.055 0.000 0.055  apimumbcms.hydtimes.com {\"msid\":\"105117134\",\"keywords\":\"null\",\"hostId\":\"53\",\"mstype\":\"3\",\"title\":\"dhanteras 2023 shopping2\",\"type\":\"dhanteras 2023 shopping2\",\"hisactive\":\"2\",\"operation\":\"add\",\"mssubtype\":\"0\",\"parentId\":\"105117040\"}
192.87.80.64 "10/Nov/2023:12:59:08 +0530" 192.87.23.116 POST "POST /api/cache/purge HTTP/1.1" 545 200 511 51  "Apache-HttpClient/4.5.2 (Java/1.8.0-262)" 127.0.0.1:3000 200 0.068 0.068 0.001 0.068  apimumbcms.hydtimes.com {\"msid\":\"105117109\",\"keywords\":\"null\",\"hostId\":\"53\",\"mstype\":\"3\",\"title\":\"dhanteras 2023 shopping4\",\"type\":\"dhanteras 2023 shopping4\",\"hisactive\":\"2\",\"operation\":\"add\",\"mssubtype\":\"0\",\"parentId\":\"105117040\"}
 "10/Nov/2023:12:59:10 +0530" 192.54.23.32 GET "GET /healthcheck HTTP/1.1" 178 200 453 2  "nginx/1.23.4 (health check server_192.87.23.870_Pool-1_http_ok)" 127.0.0.1:3000 200 0.002 0.002 0.000 0.002  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:10 +0530" 192.54.23.32 GET "GET /healthcheck HTTP/1.1" 200 200 453 2  "nginx/1.23.4 (health check server_192.87.23.870_Pool-1_https_ok)" 127.0.0.1:3000 200 0.007 0.007 0.000 0.007  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:10 +0530" 192.45.108.26 GET "GET /healthcheck HTTP/1.1" 179 200 453 2  "nginx/1.23.4 (health check server_192.45.108.199_Pool-1_http_ok)" 127.0.0.1:3000 200 0.007 0.008 0.000 0.008  apimumbcms.hydtimes.com
 "10/Nov/2023:12:59:10 +0530" 192.45.108.26 GET "GET /healthcheck HTTP/1.1" 201 200 453 2  "nginx/1.23.4 (health check server_192.45.108.199_Pool-1_https_ok)" 127.0.0.1:3000 200 0.002 0.001 0.000 0.001  apimumbcms.hydtimes.com
192.87.205.186 "10/Nov/2023:12:59:11 +0530" 192.87.23.115 GET "GET /socket.io/?EIO=4&transport=polling&t=Oku8eCj HTTP/1.1" 721 200 627 122 https://delhi.indiatimes.com/ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36" 127.0.0.1:3000 200 0.003 0.004 0.000 0.004  mumbcms.hydtimes.com
192.87.80.63 "10/Nov/2023:12:59:13 +0530" 192.87.23.114 POST "POST /api/cache/purge HTTP/1.1" 508 200 511 51  "Apache-HttpClient/4.5.2 (Java/1.8.0-262)" 127.0.0.1:3000 200 4.971 4.971 0.000 4.971  apimumbcms.hydtimes.com {\"msid\":\"97367011\",\"keywords\":\"null\",\"hostId\":\"377\",\"mstype\":\"37\",\"title\":\"Home Page News PL\",\"type\":\"null\",\"hisactive\":\"0\",\"operation\":\"PLUPDATED\",\"mssubtype\":\"1\",\"parentId\":\"97088784\"}
2 
%{NOTSPACE:remoteip}(, %{IP})* "%{HTTPDATE:timestamp}" %{IP:lbip} %{WORD:request_method} "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:request_length} %{NUMBER:status} %{NUMBER:bytes_sent} (?:%{NUMBER:body_bytes_sent}|-) %{DATA:http_referer} %{QS:http_user_agent} %{DATA:upstream_addr} %{DATA:upstream_status} %{DATA:request_time} %{DATA:upstream_response_time} %{DATA:upstream_connect_time} %{DATA:upstream_header_time} %{DATA:upstream_cache_status} %{NOTSPACE:server_name}%{GREEDYDATA:payload}

This is my grok pattern but it is not able to parse the logs which starts with double quotes. Double quotes starting lines get ignored completely.

3

If is not too late.

 grok {
	  match => { "message" => "(?:%{IPORHOST:sourceip})? \"%{HTTPDATE:time}\" %{IPORHOST:destip} %{WORD:verb} \"%{WORD:method} %{NOTSPACE:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:total} %{NUMBER:response_code} %{NUMBER:response} %{NUMBER:recive} (?:%{DATA:referrer})? \"%{DATA:user_agent}\" %{IPORHOST:[source][address]}(?::%{POSINT:[source][port]:int}) %{NUMBER:httpcode} %{NUMBER:code1} %{NUMBER:code2} %{NUMBER:code3} %{NUMBER:code4}\s+%{IPORHOST:host}" }
    }

Please review field names, they should be in /etc/nginx/nginx.conf

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.