Hello @stephenb ,
My bad,I did mistake in metricbeat.yml, your above config works fine and now I'm getting the data in my index.
Mistake done was added this: index: "cis-metric-%{+yyyy.MM.dd}",removed and worked fine with your congigs now.Thanx for you efforts on this.
Few doubts if you could help me with this:
1)I was using 7.9.1 earlier and now migrating to 8.7.1 version. I'm facing challenges ,everything worked fine in 7.9.1 and now in 8.7.1 data stream came.I'm exploring and reading documentation but unable to implement or get the data in beats indices like I want.
I've around 50 indices i.e these are such that logstash process and send data to them and these documents are frequently updated. Through documentation understood DATA STREAM are not recommended for such data and only suitable only for time series data.Correct me if I am wrong.
2)Real challenge now: I want to use metricbeat,filebeat and heartbeat(all time series data) now with my own CUSTOM templates following my own ILM policy defined.
I did this thing in 7.9.1 and worked everything well. Now in 8.7.1 what is the method and how to implement I'm not getting exactly after reading documentation.
Rollover are also not supported in 8.7.1 I guess? , and I use it in my template earlier.
Let me know how this can be done. Plz glance my Metric beat custom template and how data from beats could be mapped to custom templates?
Ofcourse I tried everything from documentation ,with no luck of data stream getting created for custom template/policy,and not sure now to achieve this.Its work in progress for now.
CRUX: I would request to help out how to send data to respective indices following custom template and custom policy. Not using by default ones.
For now with data streams seems confusing, rollover not supported ,then where could i define rollover alias type setting.
I guess without using data streams also this could be done,but did'nt got relevant docs.
I dont think below are valid for 8.7.1 
setup.ilm.rollover_alias: cis-metric
setup.ilm.pattern: '{now/d}-000001'
Metricbeat Template:(removed rollover_alias from settings in template,7.9.1 supported,8.7.1 dont support.
PUT _index_template/cis-metric
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "cis-metricbeat-policy"
},
"number_of_shards": "1",
"number_of_replicas": "0"
}
},
"mappings": {
"_source": {
"excludes": [],
"includes": [],
"enabled": true
},
"_routing": {
"required": false
},
"dynamic": true,
"numeric_detection": false,
"date_detection": true,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"properties": {
"@timestamp": {
"type": "date"
},
"agent.id": {
"type": "keyword"
},
"agent.name": {
"type": "keyword"
},
"cpu.status": {
"type": "keyword"
},
"host.name": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"metricset.name": {
"type": "keyword"
},
"metricset.period": {
"type": "long"
},
"mountpoint.details": {
"type": "keyword"
},
"process.args": {
"type": "keyword"
},
"process.details": {
"type": "keyword"
},
"process.name": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"process.pgid": {
"type": "long"
},
"process.pid": {
"type": "long"
},
"process.ppid": {
"type": "long"
},
"process.type": {
"type": "keyword"
},
"service.type": {
"type": "keyword"
},
"system.cpu.cores": {
"type": "long"
},
"system.cpu.idle.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.idle.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.iowait.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.iowait.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.irq.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.irq.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.nice.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.nice.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.softirq.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.softirq.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.steal.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.steal.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.system.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.system.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.total.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.total.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.user.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.cpu.user.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.diskio.io.ops": {
"type": "long"
},
"system.diskio.io.time": {
"type": "long"
},
"system.diskio.iostat.await": {
"type": "float"
},
"system.diskio.iostat.busy": {
"type": "float"
},
"system.diskio.iostat.queue.avg_size": {
"type": "float"
},
"system.diskio.iostat.read.await": {
"type": "float"
},
"system.diskio.iostat.read.per_sec.bytes": {
"type": "float"
},
"system.diskio.iostat.read.request.merges_per_sec": {
"type": "float"
},
"system.diskio.iostat.read.request.per_sec": {
"type": "float"
},
"system.diskio.iostat.request.avg_size": {
"type": "float"
},
"system.diskio.iostat.service_time": {
"type": "float"
},
"system.diskio.iostat.write.await": {
"type": "float"
},
"system.diskio.iostat.write.per_sec.bytes": {
"type": "float"
},
"system.diskio.iostat.write.request.merges_per_sec": {
"type": "float"
},
"system.diskio.iostat.write.request.per_sec": {
"type": "float"
},
"system.diskio.name": {
"type": "keyword"
},
"system.diskio.read.bytes": {
"type": "long"
},
"system.diskio.read.count": {
"type": "long"
},
"system.diskio.read.time": {
"type": "long"
},
"system.diskio.serial_number": {
"type": "keyword"
},
"system.diskio.write.bytes": {
"type": "long"
},
"system.diskio.write.count": {
"type": "long"
},
"system.diskio.write.time": {
"type": "long"
},
"system.diskspace.usage": {
"type": "long"
},
"system.filesystem.used.pct": {
"type": "float"
},
"system.fsstat.count": {
"type": "long"
},
"system.fsstat.status": {
"type": "keyword"
},
"system.fsstat.total_files": {
"type": "long"
},
"system.fsstat.total_size.free": {
"type": "long"
},
"system.fsstat.total_size.total": {
"type": "long"
},
"system.fsstat.total_size.used": {
"type": "long"
},
"system.load.1": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.load.15": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.load.5": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.load.cores": {
"type": "long"
},
"system.load.norm.1": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.load.norm.15": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.load.norm.5": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.memory.actual.free": {
"type": "long"
},
"system.memory.actual.used.bytes": {
"type": "long"
},
"system.memory.actual.used.pct": {
"scaling_factor": 100,
"type": "scaled_float"
},
"system.memory.free": {
"type": "long"
},
"system.memory.swap.free": {
"type": "long"
},
"system.memory.swap.total": {
"type": "long"
},
"system.memory.swap.used.bytes": {
"type": "long"
},
"system.memory.swap.used.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.memory.total": {
"type": "long"
},
"system.memory.used.bytes": {
"type": "long"
},
"system.memory.used.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.network.in.bytes": {
"type": "long"
},
"system.network.in.dropped": {
"type": "long"
},
"system.network.in.errors": {
"type": "long"
},
"system.network.in.packets": {
"type": "long"
},
"system.network.name": {
"type": "keyword"
},
"system.network.out.bytes": {
"type": "long"
},
"system.network.out.dropped": {
"type": "long"
},
"system.network.out.errors": {
"type": "long"
},
"system.network.out.packets": {
"type": "long"
},
"system.process.cpu.start_time": {
"type": "date"
},
"system.process.cpu.total.norm.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.process.cpu.total.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.process.cpu.total.value": {
"type": "long"
},
"system.process.memory.rss.bytes": {
"type": "long"
},
"system.process.memory.rss.pct": {
"scaling_factor": 1000,
"type": "scaled_float"
},
"system.process.memory.share": {
"type": "long"
},
"system.process.memory.size": {
"type": "long"
},
"system.process.state": {
"type": "keyword"
},
"user.name": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"system.process.cpu.total.ticks":
{
"type": "long"
}
}}
},
"index_patterns": [
"cis-metric-*"
],
"composed_of": []
}
system.yml(modules.d)
# Module: system
# Docs: https://www.elastic.co/guide/en/beats/metricbeat/7.8/metricbeat-module-system.html
- module: system
period: 30m
metricsets:
- memory
- module: system
period: 10s
metricsets:
- load
- network
- module: system
period: 30m
metricsets:
- fsstat
- filesystem
filesystem.ignore_types: [ssysfs, rootfs, ramfs, bdev, cgroup, cpuset,debugfs, securityfs, sockfs, dax, bpf, pipefs, anon_inodefs, configfs, devpts, hugetlbfs, autofs, pstore, mqueue, selinuxfs, rpc_pipefs, binfmt_misc, overlay,nfs4,nfs,proc,sysfs]
processors:
- script:
lang: javascript
id: MountPointDataAdapter
file: MountPointDataAdapter.js
params:
hostname: '${HOSTNAME}'
- module: system
period: 10s
metricsets:
- diskio
diskio.include_devices: ["sda", "sda1","vda","vda1"]
- module: system
period: 5m
metricsets:
- process
processes: ['.*']
process.include_cpu_ticks: true
processors:
- script:
lang: javascript
id: ProcessDataUpdater
file: ProcessDataUpdater.js
params:
hostname: '${HOSTNAME}'
- script:
lang: javascript
id: process_args_filter
file: process_args_filter.js
- module: system
period: 30m
metricsets:
- cpu
processors:
- script:
lang: javascript
id: CPUStatusUpdater.js
file: CPUStatusUpdater.js
Working metricbeat.yml for now:
metricbeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.ilm.enabled: true
setup.ilm.check_exists: true
setup.ilm.rollover_alias: cis-metric
setup.ilm.pattern: '{now/d}-000001'
setup.ilm.overwrite: false
setup.template.enabled: true
setup.template.name: "cis-metric-%{[agent.version]}"
setup.template.pattern: "cis-metric-%{[agent.version]}"
setup.template.overwrite: false
setup.ilm.policy_name: cis-metric
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
host: https://abc:5601
output.elasticsearch:
hosts: ["abc:100"]
ssl.certificate_authorities: ["/l/app/cis/monitoring/config/ece_proxy_root_ca.pem"]
protocol: ""
username: ""
password: ""
processors:
- rename:
fields:
- from: agent.hostname
to: host.name
ignore_missing: true
fail_on_error: false
- drop_fields:
fields:
- host.mac
- ecs
- agent.ephemeral_id
- agent.type
- agent.version
- agent.hostname
- event
)
,Migration is hectic.