Module nginx save documents twice with kubernetes autodiscover


(Olivier Boudet) #1

Hi,

I am trying to use Kubernetes Autodiscover with filebeat 6.4.0. It is working except my nginx log are stored twice : one in original format, and one after parsing.

For example, for this log line :

10.142.0.4 - [10.142.0.4] - - [09/Jan/2019:08:49:03 +0000] "GET /auth/realms/myProject/account HTTP/1.1" 200 128 "http://localhost:4200/depart/historique" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" 2374 0.007 c8e27131804084faac6be36e696310e0 [integration-keycloak-service-80] 10.8.1.243:8080 128 0.004 200

I have these two documents stored :

  • one not parsed :
{
  "_index": "ingress-nginx-filebeat-6.4.0-2019.01.09",
  "_type": "doc",
  "_id": "6rzNMWgBBbfCRPXqGw4Y",
  "_version": 1,
  "_score": null,
  "_source": {
    "@timestamp": "2019-01-09T08:49:03.675Z",
    "host": {
      "name": "filebeat-wrrbk"
    },
    "source": "/var/lib/docker/containers/220f40ee96b95830f397af6da967715fc7cfb98f1ed4da103bce186cea560470/220f40ee96b95830f397af6da967715fc7cfb98f1ed4da103bce186cea560470-json.log",
    "stream": "stdout",
    "beat": {
      "name": "filebeat-wrrbk",
      "hostname": "filebeat-wrrbk",
      "version": "6.4.0"
    },
    "prospector": {
      "type": "docker"
    },
    "kubernetes": {
      "labels": {
        "app": "integration-ingress-nginx",
        "pod-template-hash": "2512842583"
      },
      "pod": {
        "name": "nginx-ingress-controller-integration-6956d869d7-9brtz"
      },
      "node": {
        "name": "gke-cluster-1-pool-0-da2236b1-cvkn"
      },
      "container": {
        "name": "nginx-ingress-controller"
      },
      "namespace": "ingress-nginx",
      "replicaset": {
        "name": "nginx-ingress-controller-integration-6956d869d7"
      }
    },
    "offset": 9393024,
    "message": "10.142.0.4 - [10.142.0.4] - - [09/Jan/2019:08:49:03 +0000] \"GET /auth/realms/myProject/account HTTP/1.1\" 200 128 \"http://localhost:4200/depart/historique\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36\" 2374 0.007 c8e27131804084faac6be36e696310e0 [integration-keycloak-service-80] 10.8.1.243:8080 128 0.004 200",
    "input": {
      "type": "docker"
    }
  },
  "fields": {
    "@timestamp": [
      "2019-01-09T08:49:03.675Z"
    ]
  },
  "highlight": {
    "kubernetes.labels.app": [
      "@kibana-highlighted-field@integration@/kibana-highlighted-field@-@kibana-highlighted-field@ingress@/kibana-highlighted-field@-@kibana-highlighted-field@nginx@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1547023743675
  ]
}
  • one parsed :
{
  "_index": "ingress-nginx-filebeat-6.4.0-2019.01.09",
  "_type": "doc",
  "_id": "2rzNMWgBBbfCRPXqEw4q",
  "_version": 1,
  "_score": null,
  "_source": {
    "kubernetes": {
      "container": {
        "name": "nginx-ingress-controller"
      },
      "node": {
        "name": "gke-cluster-1-pool-0-da2236b1-cvkn"
      },
      "pod": {
        "name": "nginx-ingress-controller-integration-6956d869d7-9brtz"
      },
      "namespace": "ingress-nginx",
      "replicaset": {
        "name": "nginx-ingress-controller-integration-6956d869d7"
      },
      "labels": {
        "app": "integration-ingress-nginx",
        "pod-template-hash": "2512842583"
      }
    },
    "offset": 9380999,
    "nginx": {
      "access": {
        "response_code": "200",
        "method": "GET",
        "user_name": "[10.142.0.4] - -",
        "http_version": "1.1",
        "remote_ip_list": [
          "10.142.0.4"
        ],
        "url": "/auth/realms/myProject/account",
        "referrer": "http://localhost:4200/depart/historique",
        "request_time": "0.007",
        "remote_ip": "10.142.0.4",
        "request_length": "2374",
        "body_sent": {
          "bytes": "128"
        },
        "request_id": "c8e27131804084faac6be36e696310e0",
        "user_agent": {
          "patch": "3578",
          "major": "71",
          "minor": "0",
          "os": "Linux",
          "name": "Chrome",
          "os_name": "Linux",
          "device": "Other"
        }
      }
    },
    "prospector": {
      "type": "docker"
    },
    "read_timestamp": "2019-01-09T08:49:03.675Z",
    "source": "/var/lib/docker/containers/220f40ee96b95830f397af6da967715fc7cfb98f1ed4da103bce186cea560470/220f40ee96b95830f397af6da967715fc7cfb98f1ed4da103bce186cea560470-json.log",
    "fileset": {
      "module": "nginx",
      "name": "access"
    },
    "input": {
      "type": "docker"
    },
    "@timestamp": "2019-01-09T08:49:03.000Z",
    "stream": "stdout",
    "beat": {
      "hostname": "filebeat-wrrbk",
      "name": "filebeat-wrrbk",
      "version": "6.4.0"
    },
    "host": {
      "name": "filebeat-wrrbk"
    }
  },
  "fields": {
    "@timestamp": [
      "2019-01-09T08:49:03.000Z"
    ],
    "read_timestamp": [
      "2019-01-09T08:49:03.675Z"
    ]
  },
  "highlight": {
    "kubernetes.labels.app": [
      "@kibana-highlighted-field@integration@/kibana-highlighted-field@-@kibana-highlighted-field@ingress@/kibana-highlighted-field@-@kibana-highlighted-field@nginx@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1547023743000
  ]
}

How can I avoid that ?

My kubernetes configuration is here : https://gist.github.com/olivierboudet/b0bffbdd9148709934dfb51f13b777c6

PS : I tried to update to filebeat 6.5.4 but I have "parsing CRI timestamp" errors so I downgraded...


(Carlos PĂ©rez Aradros) #2

Hi @orgoz,

I'm checking this, is the content of filebeat-prospectors empty or did you redact it?

Best regards


(Olivier Boudet) #3

Hi @exekias

Yes, the filebeat-prospectors is empty, I am using only autodiscover.