I have a k8s node with a label of “1235679543222322113” as the key. Getting a panic when starting filebeat for collector the node info.
- Version: 7.13
- Operating System: Linux
- Steps to Reproduce:
k8s node with a label of “1235679543222322113” as the key:
[root@localhost ~]# kubectl get no 172.17.17.224 --show-labels
NAME STATUS ROLES AGE VERSION LABELS
172.17.17.224 Ready <none> xx xx 1235679543222322113=11
beat.yaml
filebeat.autodiscover:
providers:
- type: kubernetes
node: ${NODE_NAME}
hints:
enabled: true
filebeat.config.inputs:
enabled: true
path: ./configs/*.yaml
reload.enabled: true
reload.period: 10s
processors:
- add_fields:
target: cluster
fields:
name: ${CLUSTER_NAME}
- add_kubernetes_metadata:
host: ${NODE_NAME}
default_matchers.enabled: false
indexers:
- container:
- pod_uid:
matchers:
- fields:
lookup_fields: ["fields.container_id", "fields.pod_uid"]
- script:
lang: javascript
id: timestamp_filter
tag: enable
source: >
function process(event) {
var workload = event.Get("workload");
// /([01]?\d|2[0-3]):[0-5]?\d:[0-5]?\d.\d{0,9}/;
if (workload == "etcd") {
var message = event.Get("message");
var reg = /(\d{4})(\/|-)(\d{2})(\/|-)(\d{2}) (\d{2}):(\d{2}):(\d{2}).\d{0,9}/;
var res = message.match(reg);
if (res != null && res.length >= 1) {
var date = res[0];
var str = new Date(new Date(""+date).getTime() + new Date().getTimezoneOffset()*60000).toLocaleString("en-GB");
var year = str.split(", ")[0];
var time = str.split(", ")[1];
var logTime = year.split("/").map(function(i) {return i.length !==4 ?("000"+i+"").slice(-2) : i}).reverse().join("-") + "T"+time + "."+date.split(".")[1] + "Z";
event.Put("log_time",logTime);
}
}
if (workload == "kubelet") {
var message = event.Get("message");
var reg = /(\d{2}):(\d{2}):(\d{2}).\d{0,9}/;
var res = message.match(reg);
if (res != null && res.length >= 1) {
var now = new Date();
var day = now.toLocaleString().split(" ")[0]
var date = day + " " + res[0]
var str = new Date(new Date(""+date).getTime() + new Date().getTimezoneOffset()*60000).toLocaleString("en-GB");
var year = str.split(", ")[0];
var time = str.split(", ")[1];
var logTime = year.split("/").map(function(i) {return i.length !==4 ?("000"+i+"").slice(-2) : i}).reverse().join("-") + "T"+time + "."+date.split(".")[1] + "Z";
event.Put("log_time",logTime);
}
}
}
- timestamp:
field: log_time
layouts:
- '2006-01-02T15:04:05.999999999Z07:00'
test:
- '2021-08-10T19:11:18.372Z'
setup:
template.enabled: false
ilm.enabled: false
output:
elasticsearch:
index: vestack-log-write
hosts: ${ELASTIC_HOSTS}
username: ${ELASTIC_USERNAME}
password: ${ELASTIC_PASSWORD}
protocol: ${ELASTIC_PROTOCOL}
ssl:
verification_mode: none
set env:
export KUBECONFIG=/root/filebeat/kubeconfig
export NODE_NAME=172.17.17.224
export CLUSTER_NAME=xx
export ELASTIC_HOSTS=xx:9200
export ELASTIC_USERNAME=xx
export ELASTIC_PASSWORD=xx
export ELASTIC_PROTOCOL=http
start filebeat:
/root/filebeat/filebeat -e -E http.enabled=true -c /root/filebeat/beat.yml
Get panic, pretty printed stack trace:
2023-02-06T19:06:31.401+0800 INFO [autodiscover] autodiscover/autodiscover.go:113 Starting autodiscover manager
E0206 19:06:31.704363 7 runtime.go:78] Observed a panic: "makeslice: len out of range" (runtime error: makeslice: len out of range)
goroutine 181 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic(0x55a0e511a4c0, 0x55a0e562ef90)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:74 +0xa6
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:48 +0x89
panic(0x55a0e511a4c0, 0x55a0e562ef90)
/usr/local/go/src/runtime/panic.go:969 +0x1b9
github.com/elastic/go-ucfg.(*fields).setAt(0xc000c091e0, 0xf6b75ab2bc471c7, 0x55a0e5714e80, 0xc0008d0300, 0x55a0e5714520, 0xc001327000)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/ucfg.go:295 +0x71
github.com/elastic/go-ucfg.idxField.SetValue(0xf6b75ab2bc471c7, 0xc0007698c0, 0x55a0e5714e80, 0xc0008d0300, 0x55a0e5714520, 0xc001327000, 0xc000081270, 0xc000b0c8a0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/path.go:249 +0x75
github.com/elastic/go-ucfg.cfgPath.SetValue(0xc000b371d0, 0x1, 0x1, 0x55a0e4646900, 0x0, 0xc0008d0300, 0xc0007698c0, 0x55a0e5714520, 0xc001327000, 0x55a0e56f7ca0, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/path.go:229 +0x22a
github.com/elastic/go-ucfg.normalizeSetField(0xc0008d0300, 0xc0007698c0, 0x0, 0xc0012eb5c0, 0x13, 0x55a0e508bd40, 0xc000b371c0, 0x94, 0x94, 0x0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:395 +0x211
github.com/elastic/go-ucfg.normalizeMapInto(0xc0008d0300, 0xc0007698c0, 0x55a0e5489040, 0xc000d8fa10, 0x15, 0x82bd4fd3, 0x55a0e5489040)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:313 +0x238
github.com/elastic/go-ucfg.normalizeMap(0xc0007698c0, 0x55a0e5489040, 0xc000d8fa10, 0x15, 0xc000d8fa10, 0x15, 0x14)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:295 +0xaa
github.com/elastic/go-ucfg.normalizeMapValue(0xc0007698c0, 0x0, 0x0, 0x0, 0x0, 0x55a0e5489040, 0xc000d8fa10, 0x15, 0xc0007698c0, 0x55a0e5714e80, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:416 +0x52
github.com/elastic/go-ucfg.normalizeValue(0xc0007698c0, 0xc001000000, 0x0, 0x0, 0x0, 0x0, 0x55a0e508bd40, 0xc000b37080, 0x94, 0x55a0e2377e8a, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:493 +0xa2f
github.com/elastic/go-ucfg.normalizeSetField(0xc0008d01b0, 0xc0007698c0, 0x0, 0xc001398195, 0x6, 0x55a0e508bd40, 0xc000b37080, 0x94, 0x94, 0x0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:377 +0x9a
github.com/elastic/go-ucfg.normalizeMapInto(0xc0008d01b0, 0xc0007698c0, 0x55a0e5489040, 0xc000d8f9e0, 0x15, 0x55a0e52f6c60, 0x55a0e5489040)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:313 +0x238
github.com/elastic/go-ucfg.normalizeMap(0xc0007698c0, 0x55a0e5489040, 0xc000d8f9e0, 0x15, 0xc000d8f9e0, 0x15, 0x14)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:295 +0xaa
github.com/elastic/go-ucfg.normalizeMapValue(0xc0007698c0, 0x0, 0x0, 0x0, 0x0, 0x55a0e5489040, 0xc000d8f9e0, 0x15, 0xc0007698c0, 0x55a0e5714e80, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:416 +0x52
github.com/elastic/go-ucfg.normalizeValue(0xc0007698c0, 0xc001000000, 0x0, 0x0, 0x0, 0x0, 0x55a0e508bd40, 0xc000b36fa0, 0x94, 0x55a0e2377e8a, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:493 +0xa2f
github.com/elastic/go-ucfg.normalizeSetField(0xc0008d0030, 0xc0007698c0, 0x0, 0x55a0e4648473, 0x4, 0x55a0e508bd40, 0xc000b36fa0, 0x94, 0x94, 0x0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:377 +0x9a
github.com/elastic/go-ucfg.normalizeMapInto(0xc0008d0030, 0xc0007698c0, 0x55a0e5489040, 0xc000d8f950, 0x15, 0x18, 0x55a0e5489040)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:313 +0x238
github.com/elastic/go-ucfg.normalizeMap(0xc0007698c0, 0x55a0e5489040, 0xc000d8f950, 0x15, 0xc000d8f950, 0x15, 0x14)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:295 +0xaa
github.com/elastic/go-ucfg.normalizeMapValue(0xc0007698c0, 0x0, 0x0, 0x0, 0x0, 0x55a0e5489040, 0xc000d8f950, 0x15, 0x203000, 0xc0013a6ff8, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:416 +0x52
github.com/elastic/go-ucfg.normalizeValue(0xc0007698c0, 0xc001000000, 0x0, 0x0, 0x0, 0x0, 0x55a0e508bd40, 0xc000b36e30, 0x94, 0x55a0e2377e8a, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:493 +0xa2f
github.com/elastic/go-ucfg.normalizeSetField(0xc000d8ffb0, 0xc0007698c0, 0x0, 0x55a0e4654ae2, 0xa, 0x55a0e508bd40, 0xc000b36e30, 0x94, 0x94, 0x16)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:377 +0x9a
github.com/elastic/go-ucfg.normalizeMapInto(0xc000d8ffb0, 0xc0007698c0, 0x55a0e516bec0, 0xc000d8fb00, 0x15, 0xc0005ba4e0, 0x55a0e516bec0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:313 +0x238
github.com/elastic/go-ucfg.normalizeMap(0xc0007698c0, 0x55a0e516bec0, 0xc000d8fb00, 0x15, 0xc000d8fb00, 0x15, 0x14)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:295 +0xaa
github.com/elastic/go-ucfg.normalizeMapValue(0xc0007698c0, 0x0, 0x0, 0x0, 0x0, 0x55a0e516bec0, 0xc000d8fb00, 0x15, 0x203000, 0xc0013a7318, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:416 +0x52
github.com/elastic/go-ucfg.normalizeValue(0xc0007698c0, 0xc001000000, 0x0, 0x0, 0x0, 0x0, 0x55a0e508bd40, 0xc000b36e00, 0x94, 0x55a0e2377e8a, ...)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:493 +0xa2f
github.com/elastic/go-ucfg.normalizeSetField(0xc000d8ff80, 0xc0007698c0, 0x0, 0x55a0e4648133, 0x4, 0x55a0e508bd40, 0xc000b36e00, 0x94, 0x94, 0xc0004ec000)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:377 +0x9a
github.com/elastic/go-ucfg.normalizeMapInto(0xc000d8ff80, 0xc0007698c0, 0x55a0e50f0600, 0xc000d8fec0, 0x15, 0xc0013a75a8, 0x55a0e275edaf)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:313 +0x238
github.com/elastic/go-ucfg.normalizeMap(0xc0007698c0, 0x55a0e50f0600, 0xc000d8fec0, 0x15, 0x55a0e50f0600, 0x15, 0x8)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:295 +0xaa
github.com/elastic/go-ucfg.normalize(0xc0007698c0, 0x55a0e50f0600, 0xc000d8fec0, 0xc0007698c0, 0x55a0e5231420, 0xc0013a7601)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:265 +0x486
github.com/elastic/go-ucfg.(*Config).Merge(0xc000d8fef0, 0x55a0e50f0600, 0xc000d8fec0, 0x0, 0x0, 0x0, 0x55a0e22a6bcf, 0x55a0e54484c0)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/merge.go:79 +0x7a
github.com/elastic/go-ucfg.NewFrom(0x55a0e50f0600, 0xc000d8fec0, 0x0, 0x0, 0x0, 0x8, 0x8, 0x7fc4540c8108)
/go/pkg/mod/github.com/elastic/go-ucfg@v0.8.3/ucfg.go:105 +0xa5
github.com/elastic/beats/v7/libbeat/autodiscover/template.ApplyConfigTemplate(0xc000d8fb00, 0xc0011ac0c0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0)
/go/src/github.com/elastic/beats/libbeat/autodiscover/template/config.go:123 +0xc5
github.com/elastic/beats/v7/filebeat/autodiscover/builder/hints.(*logHints).CreateConfig(0xc000c06cb0, 0xc000d8fb00, 0xc0011ac090, 0x1, 0x1, 0x0, 0x0, 0x0)
/go/src/github.com/elastic/beats/filebeat/autodiscover/builder/hints/logs.go:168 +0xf46
github.com/elastic/beats/v7/libbeat/autodiscover.Builders.GetConfig(0xc000c06cc0, 0x1, 0x1, 0x55a0e56794a0, 0xc000c08c40, 0xc000d8fb00, 0xc000c08c40, 0xc000d8fa70, 0x0)
/go/src/github.com/elastic/beats/libbeat/autodiscover/builder.go:110 +0xf9
github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes.(*Provider).publish(0xc000c0c840, 0xc0011ac080, 0x1, 0x1)
/go/src/github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes/kubernetes.go:187 +0x3b3
github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes.(*pod).publishAll(0xc000c28100, 0xc00030ea80, 0x4, 0x4, 0x0)
/go/src/github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes/pod.go:529 +0xa2
github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes.(*pod).emit(0xc000c28100, 0xc001438000, 0x55a0e4649d9a, 0x5)
/go/src/github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes/pod.go:334 +0x3c5
github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes.(*pod).OnAdd(0xc000c28100, 0x55a0e55e1ca0, 0xc001438000)
/go/src/github.com/elastic/beats/libbeat/autodiscover/providers/kubernetes/pod.go:133 +0x125
github.com/elastic/beats/v7/libbeat/common/kubernetes.(*watcher).process(0xc000376930, 0x55a0e56e32c0, 0xc000c1c1c0, 0x203000)
/go/src/github.com/elastic/beats/libbeat/common/kubernetes/watcher.go:250 +0x2eb
github.com/elastic/beats/v7/libbeat/common/kubernetes.(*watcher).Start.func1()
/go/src/github.com/elastic/beats/libbeat/common/kubernetes/watcher.go:190 +0x4a
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000b362d0)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:155 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000b362d0, 0x55a0e567d320, 0xc000d8e3c0, 0x1, 0xc0011b0540)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:156 +0xad
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000b362d0, 0x3b9aca00, 0x0, 0xc000aa8301, 0xc0011b0540)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:133 +0x9a
k8s.io/apimachinery/pkg/util/wait.Until(0xc000b362d0, 0x3b9aca00, 0xc0011b0540)
/go/pkg/mod/k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:90 +0x4f
created by github.com/elastic/beats/v7/libbeat/common/kubernetes.(*watcher).Start
/go/src/github.com/elastic/beats/libbeat/common/kubernetes/watcher.go:189 +0x1de