Conditional parsing for the logs using logstash

I am able to parse the logs and send to elk but I have this requirement and would request some suggestions on it .

Application and Access logs are being written to the stdout of K8 cluster.
I am able to setup logshipper which ships the logs from stdout stream to Logstash.

Here is the format for :

Application Logs :

{
             "tags" : [
           [0] "beats_input_codec_plain_applied"
       ],
              "ecs" : {
           "version" : "1.0.0"
       },
            "agent" : {
                "version" : "7.0.1",
                   "type" : "filebeat",
               "hostname" : "eks-filebeats-2k8nm",
           "ephemeral_id" : "59c8cedb-048f-4b5f-8224-4c1fb33eac6c",
                     "id" : "be2d35b6-fa25-498b-ba7c-8af344455f93"
       },
           "stream" : "stdout",
         "@version" : "1",
       **"@timestamp" : 2021-05-31T07:40:09.835Z,**
       **"kubernetes" : {**
**                     "node" : {**
**                   "name" : "ip-10-63-21-223.ec2.internal"**
**               }**,
               "labels" : {
                  **"app_kubernetes_io/routing" : "LoadBalancer"**,
                              "helm_sh/chart" : "filedownload-management-inventory-0.1.0",
                  "app_kubernetes_io/part-of" : "filedownload-management-inventory",
                 "app_kubernetes_io/instance" : "fdi-inventory-space",
                  **"app_kubernetes_io/version" : "1.16.0"**,
                                       "repo" : "filedownload-management-inventory",
                     "app_kubernetes_io/name" : "filedownload-management-inventory",
                                        "app" : "fdi",
               "app_kubernetes_io/managed-by" : "helm",
                          "pod-template-hash" : "6f57f4b9fb",
                "app_kubernetes_io/component" : "microservice"
           },
            "container" : {
               "name" : "filedownload-management-inventory"
           },
                  **"pod" : {**
**                    "uid" : "01aa05b6-f697-4823-9d51-33c825fcd2d8",**
**                   "name" : "filedownload-management-inventory-6f57f4b9fb-x9c67"**
**               }**,
           "replicaset" : {
               "name" : "filedownload-management-inventory-6f57f4b9fb"
           },
            **"namespace" : "inventory-space"**
       },
              "log" : {
           "offset" : 0,
             "file" : {
               "path" : "/var/lib/docker/containers/504b7539bbda2e81b9c4531309e019dae9cf84f02418aa00b518f855afb3eedd/504b7539bbda2e81b9c4531309e019dae9cf84f02418aa00b518f855afb3eedd-json.log"
           }
       },
            "cloud" : {
           "availability_zone" : "us-east-1b",
                    "instance" : {
               "id" : "i-0c36649bcafb143ec"
           },
                     **"machine" : {**
**                   "type" : "m5.2xlarge"**
**               }**,
                      "region" : "us-east-1",
                    "provider" : "aws"
       },
             "host" : {
           "name" : "eks-filebeats-2k8nm"
       },
          **"message" : "{\"@message\":\"Starting Server\",\"@timestamp\":\"2021-05-31T07:40:09.833Z\",\"@fields\":{\"level\":\"debug\",\"context\":{\"file\":\"/home/app/appfiles/src/index.js\"**,
            "input" : {
           "type" : "docker"
       }
   }

Access Logs :

{
             "tags"   : [
           "beats_input_codec_plain_applied"
       ],
              "ecs"   : {
           "version"   : "1.0.0"
       },
            "agent"   : {
               "hostname"   : "eks-filebeats-dwdcj",
                "version"   : "7.0.1",
                   "type"   : "filebeat",
           "ephemeral_id"   : "6e68ec8b-c456-496e-88a8-100e31a7254f",
                     "id"   : "464fa96d-5cab-4187-9cd0-7bf132f3251f"
       },
           "stream"   : "stdout",
         "@version"   : "1",
       **"@timestamp"   : "2021-05-31T07:40:11.030Z"**,
       **"kubernetes"   : {**
**                 "node"   : {**
**               "name"   : "ip-10-63-21-47.ec2.internal"**
**           }**,
               "labels"   : {
                  **"app_kubernetes_io/routing"   : "NLB",**
                                    **"branch"    : "develop"**,
                  "app_kubernetes_io/part-of"   : "download-inventory-management",
                 "app_kubernetes_io/instance"   : "dus-test",
                                       "repo"   : "download-inventory-service",
                  **"app_kubernetes_io/version"   : "11_83b2501",**
                     "app_kubernetes_io/name"   : "inventory-service",
                                        "app"   : "mus",
               "app_kubernetes_io/managed-by"   : "Helm",
                          "pod-template-hash"   : "5794dc874",
                "app_kubernetes_io/component"   : "microservice"
           },
            **"container"   : {**
**               "name"   : "inventory-service"**
**           }**,
                  **"pod"   : {**
**                "uid"   : "51b2bc36-601e-4616-b5d9-7ee30a3d1431",**
**               "name"   : "inventory-service-5794dc874-8jqwt"**
**           }**,
           "replicaset"   : {
               "name"   : "inventory-service-5794dc874"
           },
            **"namespace"   : "test"**
       },
              "log"   : {
           "offset"   : 6000844,
             "file"   : {
               "path"   : "/var/lib/docker/containers/17e595d3a1bf3a3d25113f3fcc80f3dce16ee5902b94c1be279299225e672761/17e595d3a1bf3a3d25113f3fcc80f3dce16ee5902b94c1be279299225e672761-json.log"
           }
       },
             "host"   : {
           "name"   : "eks-filebeats-dwdcj"
       },
            "cloud"   : {
           "availability_zone"   : "us-east-1a",
                    "instance"   : {
               "id"   : "i-anjandjnjndjn313"
           },
                     "machine"   : {
               "type"   : "m5.2xlarge"
           },
                      "region"   : "us-east-1",
                    "provider"   : "aws"
       },
          **"message"   : "::ffff:10.63.20.80 - - [31/May/2021:07:40:11 +0000] "GET /metrics HTTP/1.1\" 401 61 \"-\" \"Prometheus/2.21.0-rc.0/" \"6de39360-c1e3-11eb-b364-e964a5247b5e\" \"0.240 ms**
            "input"   : {
           "type"   : "docker"
       }
   }

I am interested in only the **(starred) fields .That is, only these fields should be shipped to elastic and be at root level.
I can parse the json in application logs (message field) and text in access logs (message field).

The issue is as follows :
How can I apply one logstash config on both and ,
As per the requirement these logs should be sent to different indices, for eg all access logs should be sent to access-dd-mm-yyyy and all application logs should be sent to application-dd-mm-yyyy.

Please help .

If you only want to keep some of the top-level fields then use a prune filter with the whitelist_names option.

@Badger thank for the reply. Yes I can do that but the main issue that I experience is the points mentioned in the last which are :
How can I apply one logstash config on both (when both access and applications logs are coming out of same source) and ,
As per the requirement these logs should be sent to different indices, for eg all access logs should be sent to access-dd-mm-yyyy and all application logs should be sent to application-dd-mm-yyyy.

the access logs message field is apache format whereas the app logs are in json.

Any suggestions from anyone ?? I looked a little bit and found this might be helpful.

But how to apply conditions , I mean on what parameters shall I apply if condition in my case ?

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.