Hello!
Reference:
[Enable Elastic Security prebuilt rules and ML jobs]
So, I've uploaded ML Jobs bundled with datafeed through API using the request:
PUT _ml/anomaly_detectors/custom_rare_process_by_host_linux_ecs
{
"custom_settings" : {
"created_by" : "ml-module-security-linux",
"custom_urls" : [
{
"url_name" : "Host Details by process name",
"url_value" : "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name" : "Host Details by user name",
"url_value" : "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name" : "Hosts Overview by process name",
"url_value" : "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name" : "Hosts Overview by user name",
"url_value" : "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
}
]
},
"datafeed_config" : {
"datafeed_id" : "datafeed-custom_rare_process_by_host_linux_ecs",
"query_delay" : "87587ms",
"chunking_config" : {
"mode" : "auto"
},
"indices_options" : {
"expand_wildcards" : [
"open"
],
"ignore_unavailable" : false,
"allow_no_indices" : true,
"ignore_throttled" : true
},
"query" : {
"bool" : {
"filter" : [
{
"term" : {
"event.category" : "process"
}
},
{
"term" : {
"event.type" : "start"
}
}
],
"must" : [
{
"bool" : {
"should" : [
{
"match" : {
"host.os.type" : {
"query" : "linux",
"operator" : "OR"
}
}
},
{
"match" : {
"host.os.family" : {
"query" : "debian",
"operator" : "OR"
}
}
},
{
"match" : {
"host.os.family" : {
"query" : "redhat",
"operator" : "OR"
}
}
},
{
"match" : {
"host.os.family" : {
"query" : "suse",
"operator" : "OR"
}
}
},
{
"match" : {
"host.os.family" : {
"query" : "ubuntu",
"operator" : "OR"
}
}
}
]
}
}
]
}
},
"indices" : [
"logs-endpoint.events.*"
],
"scroll_size" : 1000,
"delayed_data_check_config" : {
"enabled" : true
}
},
"groups" : [
"security",
"endpoint",
"linux",
"process"
],
"description" : "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for processes that are unusual to a particular Linux host. Such unusual processes may indicate unauthorized services, malware, or persistence mechanisms.",
"analysis_config" : {
"bucket_span" : "15m",
"detectors" : [
{
"detector_description" : "rare process executions on Linux",
"function" : "rare",
"by_field_name" : "process.name",
"partition_field_name" : "host.name",
"detector_index" : 0
}
],
"influencers" : [
"host.name",
"process.name",
"user.name"
]
},
"analysis_limits" : {
"model_memory_limit" : "256mb",
"categorization_examples_limit" : 4
},
"data_description" : {
"time_field" : "@timestamp",
"time_format" : "epoch_ms"
},
"model_snapshot_retention_days" : 10,
"daily_model_snapshot_retention_after_days" : 1,
"results_index_name" : "shared",
"allow_lazy_open" : false
}
Than, I'm starting my feed:
POST _ml/datafeeds/datafeed-custom_rare_process_by_host_linux_ecs/_start
And opening ML job:
POST _ml/anomaly_detectors/custom_rare_process_by_host_linux_ecs/_open
Log looks like this:
The interesting thing is that I can't start/stop datafeed through UI.
As a result, I can't configure Security rule based on this ML job:
Any ideas why is this happening?
Thanks in advance!