Enable Elastic Security prebuilt ML jobs [2]

Hello!
Reference:
[Enable Elastic Security prebuilt rules and ML jobs]

So, I've uploaded ML Jobs bundled with datafeed through API using the request:

PUT _ml/anomaly_detectors/custom_rare_process_by_host_linux_ecs
{
  "custom_settings" : {
    "created_by" : "ml-module-security-linux",
    "custom_urls" : [
      {
        "url_name" : "Host Details by process name",
        "url_value" : "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
      },
      {
        "url_name" : "Host Details by user name",
        "url_value" : "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
      },
      {
        "url_name" : "Hosts Overview by process name",
        "url_value" : "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
      },
      {
        "url_name" : "Hosts Overview by user name",
        "url_value" : "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
      }
    ]
  },
  "datafeed_config" : {
    "datafeed_id" : "datafeed-custom_rare_process_by_host_linux_ecs",
    "query_delay" : "87587ms",
    "chunking_config" : {
      "mode" : "auto"
    },
    "indices_options" : {
      "expand_wildcards" : [
        "open"
      ],
      "ignore_unavailable" : false,
      "allow_no_indices" : true,
      "ignore_throttled" : true
    },
    "query" : {
      "bool" : {
        "filter" : [
          {
            "term" : {
              "event.category" : "process"
            }
          },
          {
            "term" : {
              "event.type" : "start"
            }
          }
        ],
        "must" : [
          {
            "bool" : {
              "should" : [
                {
                  "match" : {
                    "host.os.type" : {
                      "query" : "linux",
                      "operator" : "OR"
                    }
                  }
                },
                {
                  "match" : {
                    "host.os.family" : {
                      "query" : "debian",
                      "operator" : "OR"
                    }
                  }
                },
                {
                  "match" : {
                    "host.os.family" : {
                      "query" : "redhat",
                      "operator" : "OR"
                    }
                  }
                },
                {
                  "match" : {
                    "host.os.family" : {
                      "query" : "suse",
                      "operator" : "OR"
                    }
                  }
                },
                {
                  "match" : {
                    "host.os.family" : {
                      "query" : "ubuntu",
                      "operator" : "OR"
                    }
                  }
                }
              ]
            }
          }
        ]
      }
    },
    "indices" : [
      "logs-endpoint.events.*"
    ],
    "scroll_size" : 1000,
    "delayed_data_check_config" : {
      "enabled" : true
    }
  },
  "groups" : [
    "security",
    "endpoint",
    "linux",
    "process"
  ],
  "description" : "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for processes that are unusual to a particular Linux host. Such unusual processes may indicate unauthorized services, malware, or persistence mechanisms.",
  "analysis_config" : {
    "bucket_span" : "15m",
    "detectors" : [
      {
        "detector_description" : "rare process executions on Linux",
        "function" : "rare",
        "by_field_name" : "process.name",
        "partition_field_name" : "host.name",
        "detector_index" : 0
      }
    ],
    "influencers" : [
      "host.name",
      "process.name",
      "user.name"
    ]
  },
  "analysis_limits" : {
    "model_memory_limit" : "256mb",
    "categorization_examples_limit" : 4
  },
  "data_description" : {
    "time_field" : "@timestamp",
    "time_format" : "epoch_ms"
  },
  "model_snapshot_retention_days" : 10,
  "daily_model_snapshot_retention_after_days" : 1,
  "results_index_name" : "shared",
  "allow_lazy_open" : false
}

Than, I'm starting my feed:
POST _ml/datafeeds/datafeed-custom_rare_process_by_host_linux_ecs/_start

And opening ML job:
POST _ml/anomaly_detectors/custom_rare_process_by_host_linux_ecs/_open

Log looks like this:

The interesting thing is that I can't start/stop datafeed through UI.
Screenshot from 2021-11-03 17-33-37

As a result, I can't configure Security rule based on this ML job:
Screenshot from 2021-11-03 17-35-07

Any ideas why is this happening?
Thanks in advance!

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.