How to implement a custom pipeline to a index

Sir,

I have created a custom pipeline for converting the UTC time (field name : time in my log entry) to IST & created a new field ist_time. How to apply this custom pipeline to the filebeat index?

PUT _ingest/pipeline/filebeat-8.12.1_trail@custom
{
  "processors": [
    {
      "date": {
        "field": "time",
        "formats": [
          "ISO8601"
        ],
        "target_field": "ist_time",
        "timezone": "Asia/Kolkata"
      }
    }
  ]
}

Hi @Subrahmanyam_Veerank

A couple questions. There are a couple ways to do it, depending on what you're currently using trying to accomplish

You have the right idea with trying to use the @custom but that is more built into the elastic agent framework but it's still a good idea.

Can you share your file beat.yml?

Are you using a module?

If you're not using a module You should be able to put the pipeline here.

elasticsearch.output
  pipeline: filebeat-8.12.1_trail@custom
...

If you are using a module then we'll take a different approach

So what are you using?

we are not using the module sir..
i will try

elasticsearch.output
pipeline: filebeat-8.12.1_trail@custom
...

Sir,

im using the filebeat filestream and i want to convert the keyword "time" to date. Kindly guide on this.

  "_index": ".ds-filebeat-8.12.0-2024.02.12-000001",
  "_id": "i_pb1Y0BWiQLyxw44AuQ",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2024-02-23T09:46:36.622Z",
    "ecs": {
      "version": "8.0.0"
    },
    "db_main_replica_count": 0,
    "method": "GET",
    "db_count": 0,
    "input": {
      "type": "filestream"
    },
    "db_primary_count": 0,
    "pid": 341629,
    "controller": "MetricsController",
    "worker_id": "puma_1",
    "db_primary_duration_s": 0,
    "view_duration_s": 0.00062,
    "db_write_count": 0,
    "db_main_cached_count": 0,
    "db_primary_wal_cached_count": 0,
    "db_ci_replica_duration_s": 0,
    "db_cached_count": 0,
    "mem_total_bytes": 2333976,
    "rate_limiting_gates": [],
    "time": "2024-02-23T09:46:36.588Z",
    "action": "index",
    "log": {
      "offset": 5171922,
      "file": {
        "path": "/var/log/gitlab/gitlab-rails/production_json.log",
        "device_id": "64768",
        "inode": "2889074"
      }
    },
    "error": {
      "message": "Key 'log' not found",
      "type": "json"
    },
    "db_main_wal_cached_count": 0,
    "mem_objects": 3270,
    "format": "html",
    "db_ci_wal_cached_count": 0,
    "db_replica_cached_count": 0,
    "db_replica_duration_s": 0,
    "db_ci_cached_count": 0,
    "duration_s": 0.0299,
    "db_main_replica_wal_cached_count": 0,
    "db_ci_duration_s": 0,
    "db_replica_wal_count": 0,
    "db_replica_count": 0,
    "db_ci_count": 0,
    "db_main_replica_duration_s": 0,
    "agent": {
      "name": "VM150",
      "type": "filebeat",
      "version": "8.12.0",
      "ephemeral_id": "d6e2012f-bc4f-44f5-82b1-31d1320bf529",
      "id": "5384e0ab-b14d-45f2-b39d-b0e28bb41006"
    },
    "db_main_replica_cached_count": 0,
    "db_primary_wal_count": 0,
    "cpu_s": 0.037788,
    "db_main_count": 0,
    "db_ci_replica_wal_cached_count": 0,
    "mem_bytes": 2203176,
    "db_main_wal_count": 0,
    "correlation_id": "24a6fb58-ccaa-4d5a-9883-1f2e2505c452",
    "params": [],
    "db_ci_replica_count": 0,
    "db_ci_wal_count": 0,
    "path": "/-/metrics",
    "host": {
      "containerized": false,
      "ip": [
        "202.22.02.150",
        "nia0::115:5dff:fed7:7a05"
      ],
      "mac": [
        "01-65-5D-D7-7A-05"
      ],
      "hostname": "vm150",
      "architecture": "x86_64",
      "name": "vm150",
      "os": {
        "family": "debian",
        "name": "Ubuntu",
        "kernel": "5.15.0-92-generic",
        "codename": "jammy",
        "type": "linux",
        "platform": "ubuntu",
        "version": "22.04.3 LTS (Jammy Jellyfish)"
      },
      "id": "4d2b9dada0724ee2b7f7fa3396b7f27d"
    },
    "db_main_replica_wal_count": 0,
    "db_ci_replica_cached_count": 0,
    "db_replica_wal_cached_count": 0,
    "db_primary_cached_count": 0,
    "db_main_duration_s": 0,
    "db_duration_s": 0,
    "db_ci_replica_wal_count": 0,
    "mem_mallocs": 760,
    "status": 200
  },
  "fields": {
    "db_primary_count": [
      0
    ],
    "db_write_count": [
      0
    ],
    "db_primary_duration_s": [
      0
    ],
    "host.os.name.text": [
      "Ubuntu"
    ],
    "db_cached_count": [
      0
    ],
    "db_ci_cached_count": [
      0
    ],
    "db_ci_count": [
      0
    ],
    "mem_objects": [
      3270
    ],
    "pid": [
      341629
    ],
    "host.hostname": [
      "vm150"
    ],
    "host.mac": [
      "00-15-5D-D7-7A-05"
    ],
    "mem_total_bytes": [
      2333976
    ],
    "db_ci_replica_wal_cached_count": [
      0
    ],
    "path": [
      "/-/metrics"
    ],
    "db_ci_replica_count": [
      0
    ],
    "db_ci_duration_s": [
      0
    ],
    "host.os.version": [
      "22.04.3 LTS (Jammy Jellyfish)"
    ],
    "db_main_replica_wal_count": [
      0
    ],
    "host.os.name": [
      "Ubuntu"
    ],
    "action": [
      "index"
    ],
    "agent.name": [
      "VM150"
    ],
    "host.name": [
      "vm150"
    ],
    "db_duration_s": [
      0
    ],
    "db_primary_wal_count": [
      0
    ],
    "db_replica_wal_count": [
      0
    ],
    "mem_mallocs": [
      760
    ],
    "db_replica_count": [
      0
    ],
    "db_main_replica_duration_s": [
      0
    ],
    "host.os.type": [
      "linux"
    ],
    "method": [
      "GET"
    ],
    "db_ci_replica_cached_count": [
      0
    ],
    "db_main_count": [
      0
    ],
    "format": [
      "html"
    ],
    "input.type": [
      "filestream"
    ],
    "log.offset": [
      5171922
    ],
    "agent.hostname": [
      "VM150"
    ],
    "worker_id": [
      "puma_1"
    ],
    "host.architecture": [
      "x86_64"
    ],
    "db_ci_wal_cached_count": [
      0
    ],
    "db_main_replica_wal_cached_count": [
      0
    ],
    "error.type": [
      "json"
    ],
    "agent.id": [
      "5384e0ab-b14d-45f2-b39d-b0e28bb41006"
    ],
    "db_main_wal_cached_count": [
      0
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.containerized": [
      false
    ],
    "agent.version": [
      "8.12.0"
    ],
    "mem_bytes": [
      2203176
    ],
    "cpu_s": [
      0.037788
    ],
    "db_ci_replica_wal_count": [
      0
    ],
    "host.os.family": [
      "debian"
    ],
    "status": [
      200
    ],
    "db_primary_cached_count": [
      0
    ],
    "view_duration_s": [
      0.00062
    ],
    "host.ip": [
      "172.27.73.150",
      "fe80::215:5dff:fed7:7a05"
    ],
    "agent.type": [
      "filebeat"
    ],
    "db_main_replica_count": [
      0
    ],
    "db_replica_duration_s": [
      0
    ],
    "db_replica_wal_cached_count": [
      0
    ],
    "db_count": [
      0
    ],
    "db_main_wal_count": [
      0
    ],
    "host.os.kernel": [
      "5.15.0-92-generic"
    ],
    "log.file.device_id": [
      "64768"
    ],
    "db_main_cached_count": [
      0
    ],
    "db_main_replica_cached_count": [
      0
    ],
    "db_primary_wal_cached_count": [
      0
    ],
    "host.id": [
      "4d2b9dada0724ee2b7f7fa3396b7f27d"
    ],
    "controller": [
      "MetricsController"
    ],
    "host.os.codename": [
      "jammy"
    ],
    "duration_s": [
      0.0299
    ],
    "db_replica_cached_count": [
      0
    ],
    "@timestamp": [
      "2024-02-23T09:46:36.622Z"
    ],
    "host.os.platform": [
      "ubuntu"
    ],
    "error.message": [
      "Key 'log' not found"
    ],
    "log.file.inode": [
      "2889074"
    ],
    "db_ci_replica_duration_s": [
      0
    ],
    "log.file.path": [
      "/var/log/gitlab/gitlab-rails/production_json.log"
    ],
    "agent.ephemeral_id": [
      "d6e2012f-bc4f-44f5-82b1-31d1320bf529"
    ],
    "correlation_id": [
      "24a6fb58-ccaa-4d5a-9883-1f2e2505c452"
    ],
    "db_ci_wal_count": [
      0
    ],
    "time": [
      "2024-02-23T09:46:36.588Z"
    ],
    "db_main_duration_s": [
      0
    ]
  }
}

Hi @Subrahmanyam_Veerank

How do you know it's not a date? :slight_smile:

Can you run

GET .ds-filebeat-8.12.0-2024.02.12-000001/_mapping/field/time

  1. Can you look at the template and see if you will need to add a specific date mapping for that field or

  2. add Dynamic Date Detection with that proper time format

Either way ... that change will not take affect on a current index you will need to delete it or rollover to a new index

BTW I will be not available as much for then next week or so...