@stephenb yes I moved the new grok to the top pattern.
Yes, there is that same error message on every message. Here is what it looks like:
{
"@timestamp": [
"2023-12-29T16:51:38.603Z"
],
"agent.ephemeral_id": [
"0561886a-0b17-47fa-b204-f333c7117b0e"
],
"agent.hostname": [
"localhost"
],
"agent.id": [
"01b55c06-e36e-45d8-99fe-38eed50f1c0b"
],
"agent.name": [
"localhost"
],
"agent.type": [
"filebeat"
],
"agent.version": [
"8.11.2"
],
"ecs.version": [
"1.12.0"
],
"error.message": [
"field [nginx] not present as part of path [nginx.access.time]"
],
"event.created": [
"2023-12-29T16:51:38.603Z"
],
"event.dataset": [
"nginx.access"
],
"event.ingested": [
"2023-12-29T16:51:39.605Z"
],
"event.module": [
"nginx"
],
"event.original": [
"192.168.0.1 - - [29/Dec/2023:16:51:38 +0000] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\" \"-\" \"192.168.0.1\" sn=\"test.com\" rt=0.000 ua=\"-\" us=\"-\" ut=\"-\" ul=\"-\" cs=-"
],
"event.timezone": [
"+00:00"
],
"fileset.name": [
"access"
],
"host.architecture": [
"x86_64"
],
"host.containerized": [
false
],
"host.hostname": [
"localhost"
],
"host.id": [
"864c2897c3f947e58f2e627f75857003"
],
"host.ip": [
"192.168.0.1",
"2600:3c03::f03c:94ff:fe2c:258e",
"fe80::f03c:94ff:fe2c:258e"
],
"host.mac": [
"F2-3C-65-2C-25-12E"
],
"host.name": [
"localhost"
],
"host.os.codename": [
"jammy"
],
"host.os.family": [
"debian"
],
"host.os.kernel": [
"5.15.0-83-generic"
],
"host.os.name": [
"Ubuntu"
],
"host.os.name.text": [
"Ubuntu"
],
"host.os.platform": [
"ubuntu"
],
"host.os.type": [
"linux"
],
"host.os.version": [
"22.04.3 LTS (Jammy Jellyfish)"
],
"input.type": [
"log"
],
"log.file.path": [
"/var/log/nginx/access.log"
],
"log.offset": [
64683
],
"service.type": [
"nginx"
],
"source.address": [
""
],
"_id": "sjx9towBpeQAX3u1E0zV",
"_index": ".ds-filebeat-8.11.2-2023.12.08-000001",
"_score": null
}
I am not able to see the correct number of fields or events.
When I go to my filebeat-8.11.2-nginx-access-pipeline-custom and look at the processors this is what it looks like:
[
{
"grok": {
"field": "event.original",
"patterns": [
"(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"(-|%{DATA:http.request.referrer})\" \"(-|%{DATA:user_agent.original})\" \"-\" \"(-|%{IPORHOST:nginx.access.host.name})\" sn=\"(-|%{DATA:nginx.access.host.domain})\" rt=(-|%{NUMBER:nginx.access.request_time:float}) ua=\"(-|%{DATA:nginx.access.upstream_addr})\" us=\"(-|%{DATA:nginx.access.upstream_status})\" ut=\"(-|%{NUMBER:nginx.access.upstream_response_time:float})\" ul=\"(-|%{NUMBER:nginx.access.upstream_response_length:long})\" cs=-",
"(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"(-|%{DATA:http.request.referrer})\" \"(-|%{DATA:user_agent.original})\""
],
"pattern_definitions": {
"NGINX_NOTSEPARATOR": "[^\t ,:]+",
"NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*",
"NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?"
},
"ignore_missing": true
}
},
{
"set": {
"field": "event.ingested",
"value": "{{_ingest.timestamp}}"
}
},
{
"rename": {
"field": "message",
"target_field": "event.original"
}
},
{
"grok": {
"field": "nginx.access.info",
"patterns": [
"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}",
""
],
"ignore_missing": true
}
},
{
"uri_parts": {
"field": "_tmp.url_orig",
"ignore_failure": true
}
},
{
"set": {
"field": "url.domain",
"value": "{{destination.domain}}",
"if": "ctx.url?.domain == null && ctx.destination?.domain != null"
}
},
{
"remove": {
"field": [
"nginx.access.info",
"_tmp.url_orig"
],
"ignore_missing": true
}
},
{
"split": {
"separator": "\"?,?\\s+",
"ignore_missing": true,
"field": "nginx.access.remote_ip_list"
}
},
{
"split": {
"ignore_missing": true,
"field": "nginx.access.origin",
"separator": "\"?,?\\s+"
}
},
{
"set": {
"field": "source.address",
"if": "ctx.source?.address == null",
"value": ""
}
},
{
"script": {
"source": "boolean isPrivate(def dot, def ip) {\n try {\n StringTokenizer tok = new StringTokenizer(ip, dot);\n int firstByte = Integer.parseInt(tok.nextToken());\n int secondByte = Integer.parseInt(tok.nextToken());\n if (firstByte == 10) {\n return true;\n }\n if (firstByte == 192 && secondByte == 168) {\n return true;\n }\n if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) {\n return true;\n }\n if (firstByte == 127) {\n return true;\n }\n return false;\n }\n catch (Exception e) {\n return false;\n }\n} try {\n ctx.source.address = null;\n if (ctx.nginx.access.remote_ip_list == null) {\n return;\n }\n def found = false;\n for (def item : ctx.nginx.access.remote_ip_list) {\n if (!isPrivate(params.dot, item)) {\n ctx.source.address = item;\n found = true;\n break;\n }\n }\n if (!found) {\n ctx.source.address = ctx.nginx.access.remote_ip_list[0];\n }\n} catch (Exception e) {\n ctx.source.address = null;\n}",
"params": {
"dot": "."
},
"if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0",
"lang": "painless"
}
},
{
"remove": {
"field": "source.address",
"if": "ctx.source.address == null"
}
},
{
"grok": {
"ignore_failure": true,
"field": "source.address",
"patterns": [
"^%{IP:source.ip}$"
]
}
},
{
"set": {
"copy_from": "@timestamp",
"field": "event.created"
}
},
{
"date": {
"field": "nginx.access.time",
"target_field": "@timestamp",
"formats": [
"dd/MMM/yyyy:H:m:s Z"
],
"on_failure": [
{
"append": {
"value": "{{ _ingest.on_failure_message }}",
"field": "error.message"
}
}
]
}
},
{
"remove": {
"field": "nginx.access.time"
}
},
{
"user_agent": {
"ignore_missing": true,
"field": "user_agent.original"
}
},
{
"geoip": {
"field": "source.ip",
"target_field": "source.geo",
"ignore_missing": true
}
},
{
"geoip": {
"database_file": "GeoLite2-ASN.mmdb",
"field": "source.ip",
"target_field": "source.as",
"properties": [
"asn",
"organization_name"
],
"ignore_missing": true
}
},
{
"rename": {
"field": "source.as.asn",
"target_field": "source.as.number",
"ignore_missing": true
}
},
{
"rename": {
"target_field": "source.as.organization.name",
"ignore_missing": true,
"field": "source.as.organization_name"
}
},
{
"set": {
"field": "event.kind",
"value": "event"
}
},
{
"append": {
"value": "web",
"field": "event.category"
}
},
{
"append": {
"field": "event.type",
"value": "access"
}
},
{
"set": {
"value": "success",
"if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400",
"field": "event.outcome"
}
},
{
"set": {
"if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400",
"field": "event.outcome",
"value": "failure"
}
},
{
"append": {
"field": "related.ip",
"value": "{{source.ip}}",
"if": "ctx?.source?.ip != null"
}
},
{
"append": {
"field": "related.ip",
"value": "{{destination.ip}}",
"if": "ctx?.destination?.ip != null"
}
},
{
"append": {
"field": "related.user",
"value": "{{user.name}}",
"if": "ctx?.user?.name != null"
}
},
{
"script": {
"lang": "painless",
"description": "This script processor iterates over the whole document to remove fields with null values.",
"source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"
}
}
]
Failure Processors:
[
{
"set": {
"value": "{{ _ingest.on_failure_message }}",
"field": "error.message"
}
}
]
Could it have something to do with my nginx log format?
I have not used Dev Tools before, how would I go about using the test file?
As for having the nginx error fileset enabled I went ahead and set it to false, but that was not causing the error.
