Sir,
I have created a custom pipeline for converting the UTC time (field name : time in my log entry) to IST & created a new field ist_time. How to apply this custom pipeline to the filebeat index?
PUT _ingest/pipeline/filebeat-8.12.1_trail@custom
{
"processors": [
{
"date": {
"field": "time",
"formats": [
"ISO8601"
],
"target_field": "ist_time",
"timezone": "Asia/Kolkata"
}
}
]
}
A couple questions. There are a couple ways to do it, depending on what you're currently using trying to accomplish
You have the right idea with trying to use the @custom but that is more built into the elastic agent framework but it's still a good idea.
Can you share your file beat.yml?
Are you using a module?
If you're not using a module You should be able to put the pipeline here.
elasticsearch.output
pipeline: filebeat-8.12.1_trail@custom
...
If you are using a module then we'll take a different approach
So what are you using?
we are not using the module sir..
i will try
elasticsearch.output
pipeline: filebeat-8.12.1_trail@custom
...
Sir,
im using the filebeat filestream and i want to convert the keyword "time" to date. Kindly guide on this.
"_index": ".ds-filebeat-8.12.0-2024.02.12-000001",
"_id": "i_pb1Y0BWiQLyxw44AuQ",
"_version": 1,
"_score": 0,
"_source": {
"@timestamp": "2024-02-23T09:46:36.622Z",
"ecs": {
"version": "8.0.0"
},
"db_main_replica_count": 0,
"method": "GET",
"db_count": 0,
"input": {
"type": "filestream"
},
"db_primary_count": 0,
"pid": 341629,
"controller": "MetricsController",
"worker_id": "puma_1",
"db_primary_duration_s": 0,
"view_duration_s": 0.00062,
"db_write_count": 0,
"db_main_cached_count": 0,
"db_primary_wal_cached_count": 0,
"db_ci_replica_duration_s": 0,
"db_cached_count": 0,
"mem_total_bytes": 2333976,
"rate_limiting_gates": [],
"time": "2024-02-23T09:46:36.588Z",
"action": "index",
"log": {
"offset": 5171922,
"file": {
"path": "/var/log/gitlab/gitlab-rails/production_json.log",
"device_id": "64768",
"inode": "2889074"
}
},
"error": {
"message": "Key 'log' not found",
"type": "json"
},
"db_main_wal_cached_count": 0,
"mem_objects": 3270,
"format": "html",
"db_ci_wal_cached_count": 0,
"db_replica_cached_count": 0,
"db_replica_duration_s": 0,
"db_ci_cached_count": 0,
"duration_s": 0.0299,
"db_main_replica_wal_cached_count": 0,
"db_ci_duration_s": 0,
"db_replica_wal_count": 0,
"db_replica_count": 0,
"db_ci_count": 0,
"db_main_replica_duration_s": 0,
"agent": {
"name": "VM150",
"type": "filebeat",
"version": "8.12.0",
"ephemeral_id": "d6e2012f-bc4f-44f5-82b1-31d1320bf529",
"id": "5384e0ab-b14d-45f2-b39d-b0e28bb41006"
},
"db_main_replica_cached_count": 0,
"db_primary_wal_count": 0,
"cpu_s": 0.037788,
"db_main_count": 0,
"db_ci_replica_wal_cached_count": 0,
"mem_bytes": 2203176,
"db_main_wal_count": 0,
"correlation_id": "24a6fb58-ccaa-4d5a-9883-1f2e2505c452",
"params": [],
"db_ci_replica_count": 0,
"db_ci_wal_count": 0,
"path": "/-/metrics",
"host": {
"containerized": false,
"ip": [
"202.22.02.150",
"nia0::115:5dff:fed7:7a05"
],
"mac": [
"01-65-5D-D7-7A-05"
],
"hostname": "vm150",
"architecture": "x86_64",
"name": "vm150",
"os": {
"family": "debian",
"name": "Ubuntu",
"kernel": "5.15.0-92-generic",
"codename": "jammy",
"type": "linux",
"platform": "ubuntu",
"version": "22.04.3 LTS (Jammy Jellyfish)"
},
"id": "4d2b9dada0724ee2b7f7fa3396b7f27d"
},
"db_main_replica_wal_count": 0,
"db_ci_replica_cached_count": 0,
"db_replica_wal_cached_count": 0,
"db_primary_cached_count": 0,
"db_main_duration_s": 0,
"db_duration_s": 0,
"db_ci_replica_wal_count": 0,
"mem_mallocs": 760,
"status": 200
},
"fields": {
"db_primary_count": [
0
],
"db_write_count": [
0
],
"db_primary_duration_s": [
0
],
"host.os.name.text": [
"Ubuntu"
],
"db_cached_count": [
0
],
"db_ci_cached_count": [
0
],
"db_ci_count": [
0
],
"mem_objects": [
3270
],
"pid": [
341629
],
"host.hostname": [
"vm150"
],
"host.mac": [
"00-15-5D-D7-7A-05"
],
"mem_total_bytes": [
2333976
],
"db_ci_replica_wal_cached_count": [
0
],
"path": [
"/-/metrics"
],
"db_ci_replica_count": [
0
],
"db_ci_duration_s": [
0
],
"host.os.version": [
"22.04.3 LTS (Jammy Jellyfish)"
],
"db_main_replica_wal_count": [
0
],
"host.os.name": [
"Ubuntu"
],
"action": [
"index"
],
"agent.name": [
"VM150"
],
"host.name": [
"vm150"
],
"db_duration_s": [
0
],
"db_primary_wal_count": [
0
],
"db_replica_wal_count": [
0
],
"mem_mallocs": [
760
],
"db_replica_count": [
0
],
"db_main_replica_duration_s": [
0
],
"host.os.type": [
"linux"
],
"method": [
"GET"
],
"db_ci_replica_cached_count": [
0
],
"db_main_count": [
0
],
"format": [
"html"
],
"input.type": [
"filestream"
],
"log.offset": [
5171922
],
"agent.hostname": [
"VM150"
],
"worker_id": [
"puma_1"
],
"host.architecture": [
"x86_64"
],
"db_ci_wal_cached_count": [
0
],
"db_main_replica_wal_cached_count": [
0
],
"error.type": [
"json"
],
"agent.id": [
"5384e0ab-b14d-45f2-b39d-b0e28bb41006"
],
"db_main_wal_cached_count": [
0
],
"ecs.version": [
"8.0.0"
],
"host.containerized": [
false
],
"agent.version": [
"8.12.0"
],
"mem_bytes": [
2203176
],
"cpu_s": [
0.037788
],
"db_ci_replica_wal_count": [
0
],
"host.os.family": [
"debian"
],
"status": [
200
],
"db_primary_cached_count": [
0
],
"view_duration_s": [
0.00062
],
"host.ip": [
"172.27.73.150",
"fe80::215:5dff:fed7:7a05"
],
"agent.type": [
"filebeat"
],
"db_main_replica_count": [
0
],
"db_replica_duration_s": [
0
],
"db_replica_wal_cached_count": [
0
],
"db_count": [
0
],
"db_main_wal_count": [
0
],
"host.os.kernel": [
"5.15.0-92-generic"
],
"log.file.device_id": [
"64768"
],
"db_main_cached_count": [
0
],
"db_main_replica_cached_count": [
0
],
"db_primary_wal_cached_count": [
0
],
"host.id": [
"4d2b9dada0724ee2b7f7fa3396b7f27d"
],
"controller": [
"MetricsController"
],
"host.os.codename": [
"jammy"
],
"duration_s": [
0.0299
],
"db_replica_cached_count": [
0
],
"@timestamp": [
"2024-02-23T09:46:36.622Z"
],
"host.os.platform": [
"ubuntu"
],
"error.message": [
"Key 'log' not found"
],
"log.file.inode": [
"2889074"
],
"db_ci_replica_duration_s": [
0
],
"log.file.path": [
"/var/log/gitlab/gitlab-rails/production_json.log"
],
"agent.ephemeral_id": [
"d6e2012f-bc4f-44f5-82b1-31d1320bf529"
],
"correlation_id": [
"24a6fb58-ccaa-4d5a-9883-1f2e2505c452"
],
"db_ci_wal_count": [
0
],
"time": [
"2024-02-23T09:46:36.588Z"
],
"db_main_duration_s": [
0
]
}
}
How do you know it's not a date? 
Can you run
GET .ds-filebeat-8.12.0-2024.02.12-000001/_mapping/field/time
-
Can you look at the template and see if you will need to add a specific date mapping for that field or
-
add Dynamic Date Detection with that proper time format
Either way ... that change will not take affect on a current index you will need to delete it or rollover to a new index
BTW I will be not available as much for then next week or so...
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.