Hello,
I'm trying to ship the logs using filebeat(filebeat version 7.17.7 (amd64)).
my filebeat.yml file:
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/connection_status/connection.log
json.keys_under_root: true
json.add_error_key: true
json.message_key: log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
index.number_of_replicas: 1
index.codec: best_compression
setup.ilm.rollover_alias: "api-log"
setup.ilm.pattern: "{now/d}-000001"
setup.ilm.enabled: auto
name: "apigee-pp-beat2"
tags: ["apigee-mp2"]
# output.elasticsearch.ssl.verification_mode: none
output.elasticsearch:
hosts: ["DNS:9200"]
username: "user"
password: "pass"
index: "api-log-%{[agent.version]}-%{+yyyy.MM.dd}"
pipeline: geoip
protocol: "https"
ssl.certificate_authorities: ["/etc/filebeat3/CertCA.crt"]
setup.template.name: "api-log"
setup.template.pattern: "api-log-*"
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
and this is a sample of filebeat logs
2022-12-07T12:50:59.335+0300 WARN [elasticsearch] elasticsearch/client.go:414 Cannot index event publisher.Event{Content:beat.Event{Timestamp:time.Date(2022, time.December, 7, 12, 50, 58, 331497779, time.Local), Meta:null, Fields:{"agent":{"ephemeral_id":"6261d17f-efc6-474b-97bb-fe187d0a0e82","hostname":"hostname","id":"006018a2-0158-4c24-8580-4bc62a40aa8a","name":"apigee-pp-beat2","type":"filebeat","version":"7.17.7"},"cloud":{"availability_zone":"zone-1","instance":{"id":"61349517-6e17-4199-b0b2-4e337dc11fa8"},"provider":"huawei","region":"","service":{"name":"ECS"}},"domain":"domain1","ecs":{"version":"1.12.0"},"error":{"message":"Key 'log' not found","type":"json"},"host":{"architecture":"x86_64","containerized":false,"hostname":"hostname1","id":"622ba110a69e24eda2dca57e4d306baa","ip":["IP"],"mac":["02:e2:80:e9:f7:fb"],"name":"apigee-pp-beat2","os":{"codename":"Maipo","family":"redhat","kernel":"3.10.0-1160.76.1.el7.x86_64","name":"Red Hat Enterprise Linux Server","platform":"rhel","type":"linux","version":"7.9 (Maipo)"}},"input":{"type":"log"},"log":{"file":{"path":"/var/log/connection_status/connection.log"},"offset":41978},"port":"443","status":"open","tags":["apigee-mp2"]}, Private:file.State{Id:"native::20974655-64513", PrevId:"", Finished:false, Fileinfo:(*os.fileStat)(0xc0007e61a0), Source:"/var/log/connection_status/connection.log", Offset:42054, Timestamp:time.Date(2022, time.December, 7, 12, 46, 10, 227667846, time.Local), TTL:-1, Type:"log", Meta:map[string]string(nil), FileStateOS:file.StateOS{Inode:0x1400c3f, Device:0xfc01}, IdentifierName:"native"}, TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=400): {"type":"illegal_argument_exception","reason":"field [currentTimeStamp] not present as part of path [currentTimeStamp]"}, dropping event!
2022-12-07T12:50:59.335+0300 WARN [elasticsearch] elasticsearch/client.go:414 Cannot index event publisher.Event{Content:beat.Event{Timestamp:time.Date(2022, time.December, 7, 12, 50, 58, 331553901, time.Local), Meta:null, Fields:{"agent":{"ephemeral_id":"6261d17f-efc6-474b-97bb-fe187d0a0e82","hostname":"hostname","id":"006018a2-0158-4c24-8580-4bc62a40aa8a","name":"apigee-pp-beat2","type":"filebeat","version":"7.17.7"},"cloud":{"availability_zone":"zone-1","instance":{"id":"61349517-6e17-4199-b0b2-4e337dc11fa8"},"provider":"huawei","region":"","service":{"name":"ECS"}},"domain":"domain2","ecs":{"version":"1.12.0"},"error":{"message":"Key 'log' not found","type":"json"},"host":{"architecture":"x86_64","containerized":false,"hostname":"hostname1","id":"622ba110a69e24eda2dca57e4d306baa","ip":["IP"],"mac":["02:e2:80:e9:f7:fb"],"name":"apigee-pp-beat2","os":{"codename":"Maipo","family":"redhat","kernel":"3.10.0-1160.76.1.el7.x86_64","name":"Red Hat Enterprise Linux Server","platform":"rhel","type":"linux","version":"7.9 (Maipo)"}},"input":{"type":"log"},"log":{"file":{"path":"/var/log/connection_status/connection.log"},"offset":42054},"port":"443","status":"open","tags":["apigee-mp2"]}, Private:file.State{Id:"native::20974655-64513", PrevId:"", Finished:false, Fileinfo:(*os.fileStat)(0xc0007e61a0), Source:"/var/log/connection_status/connection.log", Offset:42130, Timestamp:time.Date(2022, time.December, 7, 12, 46, 10, 227667846, time.Local), TTL:-1, Type:"log", Meta:map[string]string(nil), FileStateOS:file.StateOS{Inode:0x1400c3f, Device:0xfc01}, IdentifierName:"native"}, TimeSeries:false}, Flags:0x1, Cache:publisher.EventCache{m:common.MapStr(nil)}} (status=400): {"type":"illegal_argument_exception","reason":"field [currentTimeStamp] not present as part of path [currentTimeStamp]"}, dropping event!
Thanks in advance.