filebeat logs on the node with ingress-nginx:
2021-05-01T19:16:46.125Z INFO instance/beat.go:660 Home path: [/usr/share/filebeat] Config path: [/usr/share/filebeat] Data path: [/usr/share/filebeat/data] Logs path: [/usr/share/filebeat/logs]
2021-05-01T19:16:46.211Z INFO instance/beat.go:668 Beat ID: 54be72d6-03ca-4d4b-952b-a98442d52908
2021-05-01T19:16:46.213Z INFO [api] api/server.go:62 Starting stats endpoint
2021-05-01T19:16:46.214Z INFO [seccomp] seccomp/seccomp.go:124 Syscall filter successfully installed
2021-05-01T19:16:46.214Z INFO [beat] instance/beat.go:996 Beat info {"system_info": {"beat": {"path": {"config": "/usr/share/filebeat", "data": "/usr/share/filebeat/data", "home": "/usr/share/filebeat", "logs": "/usr/share/filebeat/logs"}, "type": "filebeat", "uuid": "54be72d6-03ca-4d4b-952b-a98442d52908"}}}
2021-05-01T19:16:46.214Z INFO [beat] instance/beat.go:1005 Build info {"system_info": {"build": {"commit": "651a2ad1225f3d4420a22eba847de385b71f711d", "libbeat": "7.12.1", "time": "2021-04-20T20:58:32.000Z", "version": "7.12.1"}}}
2021-05-01T19:16:46.214Z INFO [beat] instance/beat.go:1008 Go runtime info {"system_info": {"go": {"os":"linux","arch":"amd64","max_procs":2,"version":"go1.15.9"}}}
2021-05-01T19:16:46.214Z INFO [api] api/server.go:64 Metrics endpoint listening on: 127.0.0.1:5066 (configured: localhost)
2021-05-01T19:16:46.214Z INFO [beat] instance/beat.go:1012 Host info {"system_info": {"host": {"architecture":"x86_64","boot_time":"2021-04-13T22:14:39Z","containerized":true,"name":"elastic-stack-filebeat-k5vgb","ip":["127.0.0.1/8","10.10.148.90/32"],"kernel_version":"4.14.209-160.335.amzn2.x86_64","mac":["9e:72:c4:86:20:d5"],"os":{"type":"linux","family":"redhat","platform":"centos","name":"CentOS Linux","version":"7 (Core)","major":7,"minor":9,"patch":2009,"codename":"Core"},"timezone":"UTC","timezone_offset_sec":0,"id":"5ea6a666bf6bfd4b2167796371a02dc5"}}}
2021-05-01T19:16:46.215Z INFO [beat] instance/beat.go:1041 Process info {"system_info": {"process": {"capabilities": {"inheritable":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"permitted":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"effective":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"bounding":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"ambient":null}, "cwd": "/usr/share/filebeat", "exe": "/usr/share/filebeat/filebeat", "name": "filebeat", "pid": 7, "ppid": 1, "seccomp": {"mode":"filter","no_new_privs":true}, "start_time": "2021-05-01T19:16:45.210Z"}}}
2021-05-01T19:16:46.215Z INFO instance/beat.go:304 Setup Beat: filebeat; Version: 7.12.1
2021-05-01T19:16:46.215Z INFO [index-management] idxmgmt/std.go:184 Set output.elasticsearch.index to 'filebeat-dev-4-7.12.1' as ILM is enabled.
2021-05-01T19:16:46.215Z INFO eslegclient/connection.go:99 elasticsearch url: https://elasticsearch.ops-internal.conservis.cc:443
2021-05-01T19:16:46.215Z INFO [publisher] pipeline/module.go:113 Beat name: elastic-stack-filebeat-k5vgb
2021-05-01T19:16:46.216Z INFO [monitoring] log/log.go:117 Starting metrics logging every 30s
2021-05-01T19:16:46.216Z INFO instance/beat.go:468 filebeat start running.
2021-05-01T19:16:46.217Z INFO memlog/store.go:119 Loading data file of '/usr/share/filebeat/data/registry/filebeat' succeeded. Active transaction id=5042387
2021-05-01T19:16:47.127Z INFO memlog/store.go:124 Finished loading transaction log file for '/usr/share/filebeat/data/registry/filebeat'. Active transaction id=5050005
2021-05-01T19:16:47.128Z INFO [registrar] registrar/registrar.go:109 States Loaded from registrar: 22
2021-05-01T19:16:47.128Z INFO [crawler] beater/crawler.go:71 Loading Inputs: 1
2021-05-01T19:16:47.212Z INFO add_kubernetes_metadata/kubernetes.go:71 add_kubernetes_metadata: kubernetes env detected, with version: v1.18.9-eks-d1db3c
2021-05-01T19:16:47.212Z INFO [kubernetes] kubernetes/util.go:99 kubernetes: Using node ip-10-10-153-88.ec2.internal provided in the config {"libbeat.processor": "add_kubernetes_metadata"}
2021-05-01T19:16:47.220Z INFO log/input.go:157 Configured paths: [/var/log/containers/*.log]
2021-05-01T19:16:47.220Z INFO [crawler] beater/crawler.go:141 Starting input (ID: 3344727458180118799)
2021-05-01T19:16:47.221Z INFO [crawler] beater/crawler.go:108 Loading and starting Inputs completed. Enabled inputs: 1
2021-05-01T19:16:47.222Z INFO [autodiscover.pod] kubernetes/util.go:114 kubernetes: Using pod name elastic-stack-filebeat-k5vgb and namespace default to discover kubernetes node
2021-05-01T19:16:47.222Z INFO cfgfile/reload.go:164 Config reloader started
2021-05-01T19:16:47.222Z INFO cfgfile/reload.go:224 Loading of config files completed.
2021-05-01T19:16:47.223Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/kube-asg-node-drainer-d8tnd_kube-system_kube-node-drainer-cc2b2cc2d35605e31998d0db4a2941537048cc7c8cfe41c7c20a3d5c20cb2ead.log
2021-05-01T19:16:47.223Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/kubeservis-core-kube2iam-vmdpl_kubeservis_kube2iam-1c25b5cf40f65c2ed76698628cf894b6808e0bd0dc12db023b85be7b24c250ae.log
2021-05-01T19:16:47.224Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/elastic-stack-metricbeat-jd2rs_default_metricbeat-e931fad9ac076a340b53a5a2942ccf3358c7b3d957a617c2c588e13d43a03ca4.log
2021-05-01T19:16:47.313Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/kubeservis-core-aws-cluster-autoscaler-6bdcd566d7-qc7vx_kubeservis_aws-cluster-autoscaler-da3adb0437090071d601d7a558ed68d0ef1635169c3a9221feba41a438a69dd2.log
2021-05-01T19:16:47.314Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/qatool-qa-env-app-7f67578b6f-8tdjw_dev_kubeservis-app-97cb36cf4f148c8748d4f78d632d811c59188b282f3871e724618c77072ce084.log
2021-05-01T19:16:47.319Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/elastic-stack-filebeat-k5vgb_default_filebeat-65ed8d613d301683549af7bb50dba1261be4d2243cf9507ad1a9b75f4b30acf7.log
2021-05-01T19:16:47.413Z INFO [autodiscover.pod] kubernetes/util.go:120 kubernetes: Using node ip-10-10-153-88.ec2.internal discovered by in cluster pod node query
2021-05-01T19:16:47.414Z INFO [autodiscover] autodiscover/autodiscover.go:113 Starting autodiscover manager
2021-05-01T19:16:47.727Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.728Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.728Z INFO log/input.go:157 Configured paths: [/var/log/nginx/error.log*]
2021-05-01T19:16:47.731Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.731Z INFO log/input.go:157 Configured paths: [/var/log/nginx/error.log*]
2021-05-01T19:16:47.732Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.734Z INFO log/input.go:157 Configured paths: [/var/log/nginx/error.log*]
2021-05-01T19:16:47.735Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.735Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.737Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.737Z INFO log/input.go:157 Configured paths: [/var/log/nginx/error.log*]
2021-05-01T19:16:47.738Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.812Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.813Z INFO log/input.go:157 Configured paths: [/var/log/nginx/access.log*]
2021-05-01T19:16:47.813Z INFO log/input.go:157 Configured paths: [/var/log/nginx/error.log*]
2021-05-01T19:16:47.813Z INFO eslegclient/connection.go:99 elasticsearch url: https://elasticsearch.ops-internal.conservis.cc:443
2021-05-01T19:16:48.029Z INFO [esclientleg] eslegclient/connection.go:314 Attempting to connect to Elasticsearch version 7.12.1
2021-05-01T19:16:48.324Z INFO [publisher_pipeline_output] pipeline/output.go:143 Connecting to backoff(elasticsearch(https://elasticsearch.ops-internal.conservis.cc:443))
2021-05-01T19:16:48.324Z INFO [publisher] pipeline/retry.go:219 retryer: send unwait signal to consumer
2021-05-01T19:16:48.324Z INFO [publisher] pipeline/retry.go:223 done
2021-05-01T19:16:48.340Z INFO [esclientleg] eslegclient/connection.go:314 Attempting to connect to Elasticsearch version 7.12.1
2021-05-01T19:16:48.363Z INFO [license] licenser/es_callback.go:51 Elasticsearch license: Basic
2021-05-01T19:16:48.371Z INFO [esclientleg] eslegclient/connection.go:314 Attempting to connect to Elasticsearch version 7.12.1
2021-05-01T19:16:48.416Z INFO [index-management] idxmgmt/std.go:261 Auto ILM enable success.
2021-05-01T19:16:48.842Z INFO [index-management] idxmgmt/std.go:274 ILM policy successfully loaded.
2021-05-01T19:16:48.842Z INFO [index-management] idxmgmt/std.go:407 Set setup.template.name to '{filebeat-dev-4-7.12.1 {now/d}-000001}' as ILM is enabled.
2021-05-01T19:16:48.842Z INFO [index-management] idxmgmt/std.go:412 Set setup.template.pattern to 'filebeat-dev-4-7.12.1-*' as ILM is enabled.
2021-05-01T19:16:48.842Z INFO [index-management] idxmgmt/std.go:446 Set settings.index.lifecycle.rollover_alias in template to {filebeat-dev-4-7.12.1 {now/d}-000001} as ILM is enabled.
2021-05-01T19:16:48.842Z INFO [index-management] idxmgmt/std.go:450 Set settings.index.lifecycle.name in template to {filebeat {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"30d","max_size":"50gb"}}}}}}} as ILM is enabled.
2021-05-01T19:16:48.846Z INFO template/load.go:97 Template filebeat-dev-4-7.12.1 already exists and will not be overwritten.
2021-05-01T19:16:48.846Z INFO [index-management] idxmgmt/std.go:298 Loaded index template.
2021-05-01T19:16:48.851Z INFO [index-management] idxmgmt/std.go:309 Write alias successfully generated.
2021-05-01T19:16:48.854Z INFO [publisher_pipeline_output] pipeline/output.go:151 Connection to backoff(elasticsearch(https://elasticsearch.ops-internal.conservis.cc:443)) established
2021-05-01T19:16:57.324Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/kubeservis-core-ingress-nginx-controller-57d5c89b77-f44fv_kubeservis_controller-d6629b38269d9467350e1f23416072811e78f2a9ed90c95560176964f3a39a1b.log
2021-05-01T19:16:57.325Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/aws-node-mrm5d_kube-system_aws-node-ff7b7b9d9d20efc7cc40219ba6afbbe125053a1980bf464fe40237bc98df7707.log
2021-05-01T19:17:16.220Z INFO [monitoring] log/log.go:144 Non-zero metrics in the last 30s {"monitoring": {"metrics": {"beat":{"cgroup":{"cpu":{"cfs":{"period":{"us":100000},"quota":{"us":19900}},"id":"65ed8d613d301683549af7bb50dba1261be4d2243cf9507ad1a9b75f4b30acf7","stats":{"periods":114,"throttled":{"ns":4036236003,"periods":39}}},"cpuacct":{"id":"65ed8d613d301683549af7bb50dba1261be4d2243cf9507ad1a9b75f4b30acf7","total":{"ns":998101804}},"memory":{"id":"65ed8d613d301683549af7bb50dba1261be4d2243cf9507ad1a9b75f4b30acf7","mem":{"limit":{"bytes":208666624},"usage":{"bytes":35082240}}}},"cpu":{"system":{"ticks":80,"time":{"ms":82}},"total":{"ticks":570,"time":{"ms":575},"value":570},"user":{"ticks":490,"time":{"ms":493}}},"handles":{"limit":{"hard":1048576,"soft":1048576},"open":21},"info":{"ephemeral_id":"f9d13f71-d924-453a-b9ce-cd71bf55dc1a","uptime":{"ms":30477}},"memstats":{"gc_next":25855440,"memory_alloc":21435648,"memory_sys":75449344,"memory_total":100017048,"rss":85090304},"runtime":{"goroutines":152}},"filebeat":{"events":{"added":428,"done":428},"harvester":{"open_files":8,"running":8,"started":8}},"libbeat":{"config":{"module":{"running":1,"starts":1},"reloads":1,"scans":1},"output":{"events":{"acked":397,"active":0,"batches":17,"total":397},"read":{"bytes":19173},"type":"elasticsearch","write":{"bytes":669799}},"pipeline":{"clients":4,"events":{"active":0,"filtered":31,"published":397,"retry":50,"total":428},"queue":{"acked":397}}},"registrar":{"states":{"cleanup":1,"current":22,"update":428},"writes":{"success":48,"total":48}},"system":{"cpu":{"cores":2},"load":{"1":1.07,"15":0.35,"5":0.45,"norm":{"1":0.535,"15":0.175,"5":0.225}}}}}}
2021-05-01T19:17:17.326Z INFO log/harvester.go:302 Harvester started for file: /var/log/containers/kubeservis-core-external-dns-cc9987f97-fv22l_kubeservis_external-dns-2ddaf4741335e56865af3954f93a4724b333347af632c8b51dd3567e8d8e14c8.log