Warm data nodes receiving documents without warm data ILM

Elastic Stack Elasticsearch
1

Today I added 2 warm data nodes to our 3 node cluster. This is the first time we are using anything besides hot nodes so I have some observations and questions.

The 3 hot data nodes have roles as below

     "roles" : [
        "data_content",
        "data_hot",
        "ingest",
        "master",
        "ml",
        "transform"
      ]

The 2 warm nodes have roles as below

      "roles" : [
        "data_warm"
      ]

None of our ILM policies have warm phases configured but the warm nodes have a growing document count eg.

      "indices" : {
        "docs" : {
          "count" : 299173140,
          "deleted" : 0
        }
      }

The growth is sporadic since the new nodes were added. Is this expected and will I be able to see what kind of documents/indices are finding their way onto these nodes?

Thanks in advance!

2

Further investigation shows that my warm nodes are getting primary and replica shards.

3

What does your ILM policies and index templates look like?

4

Hi Christian

Thanks for the response.

Default winlogbeat

_ilm/policy/winlogbeat
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "30d"
          },
          "set_priority": {
            "priority": 100
          }
        }
      }
    }
  }
}
5 
{
  "index": {
    "lifecycle": {
      "name": "winlogbeat",
      "rollover_alias": "winlogbeat-7.9.3"
    },
    "mapping": {
      "total_fields": {
        "limit": "10000"
      }
    },
    "refresh_interval": "5s",
    "number_of_shards": "1",
    "max_docvalue_fields_search": "200",
    "query": {
      "default_field": [
        "message",
        "tags",
        "agent.ephemeral_id",
        "agent.id",
        "agent.name",
        "agent.type",
        "agent.version",
        "as.organization.name",
        "client.address",
        "client.as.organization.name",
        ...
        "geo.city_name",
        "geo.continent_name",
        "geo.country_iso_code",
        "geo.country_name",
        "geo.name",
        "geo.region_iso_code",
        "geo.region_name",
        "group.domain",
        "group.id",
        "group.name",
        "hash.md5",
        "hash.sha1",
        "hash.sha256",
        "hash.sha512",
        "host.architecture",
        "host.geo.city_name",
        "host.geo.continent_name",
        "host.geo.country_iso_code",
        "host.geo.country_name",
        "host.geo.name",
        "host.geo.region_iso_code",
        "host.geo.region_name",
        "host.hostname",
        "host.id",
        "host.mac",
        "host.name",
        "host.os.family",
        "host.os.full",
        "host.os.kernel",
        "host.os.name",
        "host.os.platform",
        "host.os.version",
        "host.type",
        "host.user.domain",
        "host.user.email",
        "host.user.full_name",
        "host.user.group.domain",
        "host.user.group.id",
        "host.user.group.name",
        "host.user.hash",
        "host.user.id",
        "host.user.name",
        "http.request.body.content",
        "http.request.method",
        "http.request.referrer",
        "http.response.body.content",
        "http.version",
        "log.level",
        "log.logger",
        "log.origin.file.name",
        "log.origin.function",
        "log.original",
        "log.syslog.facility.name",
        "log.syslog.severity.name",
        "network.application",
        "network.community_id",
        "network.direction",
        "network.iana_number",
        "network.name",
        "network.protocol",
        "network.transport",
        "network.type",
        "observer.geo.city_name",
        "observer.geo.continent_name",
        "observer.geo.country_iso_code",
        "observer.geo.country_name",
        "observer.geo.name",
        "observer.geo.region_iso_code",
        "observer.geo.region_name",
        "observer.hostname",
        "observer.mac",
        "observer.name",
        "observer.os.family",
        "observer.os.full",
        "observer.os.kernel",
        "observer.os.name",
        "observer.os.platform",
        "observer.os.version",
        "observer.product",
        "observer.serial_number",
        "observer.type",
        "observer.vendor",
        "observer.version",
        "organization.id",
        "organization.name",
        "os.family",
        "os.full",
        "os.kernel",
        "os.name",
        "os.platform",
        "os.version",
        "package.architecture",
        "package.checksum",
        "package.description",
        "package.install_scope",
        "package.license",
        "package.name",
        "package.path",
        "package.version",
        "process.args",
        "text",
        "process.executable",
        "process.hash.md5",
        "process.hash.sha1",
        "process.hash.sha256",
        "process.hash.sha512",
        "process.name",
        "text",
        "text",
        "text",
        "text",
        "text",
        "process.thread.name",
        "process.title",
        "process.working_directory",
        "server.address",
        "server.as.organization.name",
        "server.domain",
        "server.geo.city_name",
        "server.geo.continent_name",
        "server.geo.country_iso_code",
        "server.geo.country_name",
        "server.geo.name",
        "server.geo.region_iso_code",
        "server.geo.region_name",
        "server.mac",
        "server.registered_domain",
        "server.top_level_domain",
        "server.user.domain",
        "server.user.email",
        "server.user.full_name",
        "server.user.group.domain",
        "server.user.group.id",
        "server.user.group.name",
        "server.user.hash",
        "server.user.id",
        "server.user.name",
        "service.ephemeral_id",
        "service.id",
        "service.name",
        "service.node.name",
        "service.state",
        "service.type",
        "service.version",
        "source.address",
        "source.as.organization.name",
        "source.domain",
        "source.geo.city_name",
        "source.geo.continent_name",
        "source.geo.country_iso_code",
        "source.geo.country_name",
        "source.geo.name",
        "source.geo.region_iso_code",
        "source.geo.region_name",
        "source.mac",
        "source.registered_domain",
        "source.top_level_domain",
        "source.user.domain",
        "source.user.email",
        "source.user.full_name",
        "source.user.group.domain",
        "source.user.group.id",
        "source.user.group.name",
        "source.user.hash",
        "source.user.id",
        "source.user.name",
        "threat.framework",
        "threat.tactic.id",
        "threat.tactic.name",
        "threat.tactic.reference",
        "threat.technique.id",
        "threat.technique.name",
        "threat.technique.reference",
        "tracing.trace.id",
        "tracing.transaction.id",
        "url.domain",
        "url.extension",
        "url.fragment",
        "url.full",
        "url.original",
        "url.password",
        "url.path",
        "url.query",
        "url.registered_domain",
        "url.scheme",
        "url.top_level_domain",
        "url.username",
        "user.domain",
        "user.email",
        "user.full_name",
        "user.group.domain",
        "user.group.id",
        "user.group.name",
        "user.hash",
        "user.id",
        "user.name",
        "user_agent.device.name",
        "user_agent.name",
        "text",
        "user_agent.original",
        "user_agent.os.family",
        "user_agent.os.full",
        "user_agent.os.kernel",
        "user_agent.os.name",
        "user_agent.os.platform",
        "user_agent.os.version",
        "user_agent.version",
        "text",
        "agent.hostname",
        "timeseries.instance",
        "cloud.project.id",
        "cloud.image.id",
        "host.os.build",
        "host.os.codename",
        "kubernetes.pod.name",
        "kubernetes.pod.uid",
        "kubernetes.namespace",
        "kubernetes.node.name",
        "kubernetes.replicaset.name",
        "kubernetes.deployment.name",
        "kubernetes.statefulset.name",
        "kubernetes.container.name",
        "kubernetes.container.image",
        "jolokia.agent.version",
        "jolokia.agent.id",
        "jolokia.server.product",
        "jolokia.server.version",
        "jolokia.server.vendor",
        "jolokia.url",
        "log.file.path",
        "event.original",
        "winlog.api",
        "winlog.activity_id",
        "winlog.computer_name",
        "winlog.event_data.AuthenticationPackageName",
        "winlog.event_data.Binary",
        "winlog.event_data.BitlockerUserInputTime",
        "winlog.event_data.BootMode",
        "winlog.event_data.BootType",
        "winlog.event_data.BuildVersion",
        "winlog.event_data.Company",
        "winlog.event_data.CorruptionActionState",
        "winlog.event_data.CreationUtcTime",
        "winlog.event_data.Description",
        "winlog.event_data.Detail",
        "winlog.event_data.DeviceName",
        "winlog.event_data.DeviceNameLength",
        "winlog.event_data.DeviceTime",
        "winlog.event_data.DeviceVersionMajor",
        "winlog.event_data.DeviceVersionMinor",
        "winlog.event_data.DriveName",
        "winlog.event_data.DriverName",
        "winlog.event_data.DriverNameLength",
        "winlog.event_data.DwordVal",
        "winlog.event_data.EntryCount",
        "winlog.event_data.ExtraInfo",
        "winlog.event_data.FailureName",
        "winlog.event_data.FailureNameLength",
        "winlog.event_data.FileVersion",
        "winlog.event_data.FinalStatus",
        "winlog.event_data.Group",
        "winlog.event_data.IdleImplementation",
        "winlog.event_data.IdleStateCount",
        "winlog.event_data.ImpersonationLevel",
        "winlog.event_data.IntegrityLevel",
        "winlog.event_data.IpAddress",
        "winlog.event_data.IpPort",
        "winlog.event_data.KeyLength",
        "winlog.event_data.LastBootGood",
        "winlog.event_data.LastShutdownGood",
        "winlog.event_data.LmPackageName",
        "winlog.event_data.LogonGuid",
        "winlog.event_data.LogonId",
        "winlog.event_data.LogonProcessName",
        "winlog.event_data.LogonType",
        "winlog.event_data.MajorVersion",
        "winlog.event_data.MaximumPerformancePercent",
        "winlog.event_data.MemberName",
        "winlog.event_data.MemberSid",
        "winlog.event_data.MinimumPerformancePercent",
        "winlog.event_data.MinimumThrottlePercent",
        "winlog.event_data.MinorVersion",
        "winlog.event_data.NewProcessId",
        "winlog.event_data.NewProcessName",
        "winlog.event_data.NewSchemeGuid",
        "winlog.event_data.NewTime",
        "winlog.event_data.NominalFrequency",
        "winlog.event_data.Number",
        "winlog.event_data.OldSchemeGuid",
        "winlog.event_data.OldTime",
        "winlog.event_data.OriginalFileName",
        "winlog.event_data.Path",
        "winlog.event_data.PerformanceImplementation",
        "winlog.event_data.PreviousCreationUtcTime",
        "winlog.event_data.PreviousTime",
        "winlog.event_data.PrivilegeList",
        "winlog.event_data.ProcessId",
        "winlog.event_data.ProcessName",
        "winlog.event_data.ProcessPath",
        "winlog.event_data.ProcessPid",
        "winlog.event_data.Product",
        "winlog.event_data.PuaCount",
        "winlog.event_data.PuaPolicyId",
        "winlog.event_data.QfeVersion",
        "winlog.event_data.Reason",
        "winlog.event_data.SchemaVersion",
        "winlog.event_data.ScriptBlockText",
        "winlog.event_data.ServiceName",
        "winlog.event_data.ServiceVersion",
        "winlog.event_data.ShutdownActionType",
        "winlog.event_data.ShutdownEventCode",
        "winlog.event_data.ShutdownReason",
        "winlog.event_data.Signature",
        "winlog.event_data.SignatureStatus",
        "winlog.event_data.Signed",
        "winlog.event_data.StartTime",
        "winlog.event_data.State",
        "winlog.event_data.Status",
        "winlog.event_data.StopTime",
        "winlog.event_data.SubjectDomainName",
        "winlog.event_data.SubjectLogonId",
        "winlog.event_data.SubjectUserName",
        "winlog.event_data.SubjectUserSid",
        "winlog.event_data.TSId",
        "winlog.event_data.TargetDomainName",
        "winlog.event_data.TargetInfo",
        "winlog.event_data.TargetLogonGuid",
        "winlog.event_data.TargetLogonId",
        "winlog.event_data.TargetServerName",
        "winlog.event_data.TargetUserName",
        "winlog.event_data.TargetUserSid",
        "winlog.event_data.TerminalSessionId",
        "winlog.event_data.TokenElevationType",
        "winlog.event_data.TransmittedServices",
        "winlog.event_data.UserSid",
        "winlog.event_data.Version",
        "winlog.event_data.Workstation",
        "winlog.event_data.param1",
        "winlog.event_data.param2",
        "winlog.event_data.param3",
        "winlog.event_data.param4",
        "winlog.event_data.param5",
        "winlog.event_data.param6",
        "winlog.event_data.param7",
        "winlog.event_data.param8",
        "winlog.event_id",
        "winlog.keywords",
        "winlog.channel",
        "winlog.record_id",
        "winlog.related_activity_id",
        "winlog.opcode",
        "winlog.provider_guid",
        "winlog.provider_name",
        "winlog.task",
        "winlog.user.identifier",
        "winlog.user.name",
        "winlog.user.domain",
        "winlog.user.type",
        "fields.*"
      ]
    }
  }
}
6

Something is very strange I have a primary shard on the warm node with a replica on hot.

7

Which version of Elasticsearch are you using?

8

{
...
"version" : {
"number" : "7.14.0",
"build_flavor" : "default",
"build_type" : "deb",
"build_hash" : "dd5a0a2acaa2045ff9624f3729fc8a6f40835aa1",
"build_date" : "2021-07-29T20:49:32.864135063Z",
"build_snapshot" : false,
"lucene_version" : "8.9.0",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},

9

What is the output of the cat nodes API?

10 
10.244.101.61 33 74  1 0.32 0.18 0.13 l      - sec-kibana1.t3.nz
10.244.101.56 58 99  1 0.00 0.01 0.00 w      - sec-elastic-warm1.t3.nz
10.244.101.53 47 99 22 1.08 1.17 1.15 hilmst - sec-elastic3.t3.nz
10.244.101.52 17 95 10 0.46 0.46 0.51 hilmst - sec-elastic2.t3.nz
10.244.101.51 20 93 23 1.35 1.48 1.43 hilmst * sec-elastic1.t3.nz
10.244.101.57 44 99  1 0.14 0.07 0.09 w      - sec-elastic-warm2.t3.nz
11

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.