Hi,
I have tried to enable native realm authentication in elasticsearch 7.3.0 but the problem i have noticed that I could see missing values in .security index mapping when ES cluster is running without kibana. As a result i could not able to create any user / roles by using API.
log
[2020-02-24T11:43:33,609][INFO ][o.e.x.s.s.SecurityIndexManager] Missing _meta field in mapping [_doc] of index [.security]
[2020-02-24T11:43:33,609][WARN ][o.e.c.s.ClusterApplierService] failed to notify ClusterStateListener
java.lang.IllegalStateException: Cannot read security-version string in index .security
at org.elasticsearch.xpack.security.support.SecurityIndexManager.readMappingVersion(SecurityIndexManager.java:320) ~[?:?]
I have tested the same ES configuration with kibana pointed to ES. I didn't face issues.
.security index mapping without kibana
{
".security-7" : {
"aliases" : {
".security" : { }
},
"mappings" : {
"dynamic_templates" : [
{
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"norms" : false,
"type" : "text"
}
}
},
{
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"fields" : {
"keyword" : {
"ignore_above" : 256,
"type" : "keyword"
}
},
"norms" : false,
"type" : "text"
}
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "keyword"
},
"actions" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"application" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"geoip" : {
"dynamic" : "true",
"properties" : {
"ip" : {
"type" : "ip"
},
"latitude" : {
"type" : "half_float"
},
"location" : {
"type" : "geo_point"
},
"longitude" : {
"type" : "half_float"
}
}
},
"indices" : {
"properties" : {
"allow_restricted_indices" : {
"type" : "boolean"
},
"names" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"privileges" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"metadata" : {
"type" : "object"
},
"name" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"type" : {
"type" : "text",
"norms" : false,
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"settings" : {
"index" : {
"refresh_interval" : "5s",
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1",
"provided_name" : ".security-7",
"format" : "6",
"creation_date" : "1582532501252",
"analysis" : {
"filter" : {
"email" : {
"type" : "pattern_capture",
"preserve_original" : "true",
"patterns" : [
"([^@]+)",
"(\p{L}+)",
"(\d+)",
"@(.+)"
]
}
},
"analyzer" : {
"email" : {
"filter" : [
"email",
"lowercase",
"unique"
],
"tokenizer" : "uax_url_email"
}
}
},
"priority" : "1000",
"number_of_replicas" : "1",
"uuid" : "Nv2KiMKDRfW8UgWL1hANbw",
"version" : {
"created" : "7030099"
}
}
}
}
}
.security index mapping with kibana
{
".security-7" : {
"aliases" : {
".security" : { }
},
"mappings" : {
"dynamic" : "strict",
"_meta" : {
"security-version" : "7.3.0"
},
"properties" : {
"access_token" : {
"properties" : {
"invalidated" : {
"type" : "boolean"
},
"realm" : {
"type" : "keyword"
},
"user_token" : {
"properties" : {
"authentication" : {
"type" : "binary"
},
"expiration_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"id" : {
"type" : "keyword"
},
"metadata" : {
"type" : "object",
"dynamic" : "false"
},
"version" : {
"type" : "integer"
}
}
}
}
},
"actions" : {
"type" : "keyword"
},
"api_key_hash" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"api_key_invalidated" : {
"type" : "boolean"
},
"application" : {
"type" : "keyword"
},
"applications" : {
"properties" : {
"application" : {
"type" : "keyword"
},
"privileges" : {
"type" : "keyword"
},
"resources" : {
"type" : "keyword"
}
}
},
"cluster" : {
"type" : "keyword"
},
"creation_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"creator" : {
"properties" : {
"metadata" : {
"type" : "object",
"dynamic" : "false"
},
"principal" : {
"type" : "keyword"
},
"realm" : {
"type" : "keyword"
}
}
},
"doc_type" : {
"type" : "keyword"
},
"email" : {
"type" : "text",
"analyzer" : "email"
},
"enabled" : {
"type" : "boolean"
},
"expiration_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"full_name" : {
"type" : "text"
},
"global" : {
"properties" : {
"application" : {
"properties" : {
"manage" : {
"properties" : {
"applications" : {
"type" : "keyword"
}
}
}
}
}
}
},
"indices" : {
"properties" : {
"allow_restricted_indices" : {
"type" : "boolean"
},
"field_security" : {
"properties" : {
"except" : {
"type" : "keyword"
},
"grant" : {
"type" : "keyword"
}
}
},
"names" : {
"type" : "keyword"
},
"privileges" : {
"type" : "keyword"
},
"query" : {
"type" : "keyword"
}
}
},
"limited_by_role_descriptors" : {
"type" : "object",
"enabled" : false
},
"metadata" : {
"type" : "object",
"dynamic" : "false"
},
"name" : {
"type" : "keyword"
},
"password" : {
"type" : "keyword",
"index" : false,
"doc_values" : false
},
"refresh_token" : {
"properties" : {
"client" : {
"properties" : {
"realm" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
},
"user" : {
"type" : "keyword"
}
}
},
"invalidated" : {
"type" : "boolean"
},
"refresh_time" : {
"type" : "date",
"format" : "epoch_millis"
},
"refreshed" : {
"type" : "boolean"
},
"superseding" : {
"properties" : {
"encrypted_tokens" : {
"type" : "binary"
},
"encryption_iv" : {
"type" : "binary"
},
"encryption_salt" : {
"type" : "binary"
}
}
},
"token" : {
"type" : "keyword"
}
}
},
"role_descriptors" : {
"type" : "object",
"enabled" : false
},
"role_templates" : {
"properties" : {
"format" : {
"type" : "keyword"
},
"template" : {
"type" : "text"
}
}
},
"roles" : {
"type" : "keyword"
},
"rules" : {
"type" : "object",
"dynamic" : "false"
},
"run_as" : {
"type" : "keyword"
},
"type" : {
"type" : "keyword"
},
"username" : {
"type" : "keyword"
},
"version" : {
"type" : "integer"
}
}
},
"settings" : {
"index" : {
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1",
"provided_name" : ".security-7",
"format" : "6",
"creation_date" : "1566300744986",
"analysis" : {
"filter" : {
"email" : {
"type" : "pattern_capture",
"preserve_original" : "true",
"patterns" : [
"([^@]+)",
"(\p{L}+)",
"(\d+)",
"@(.+)"
]
}
},
"analyzer" : {
"email" : {
"filter" : [
"email",
"lowercase",
"unique"
],
"tokenizer" : "uax_url_email"
}
}
},
"priority" : "1000",
"number_of_replicas" : "1",
"uuid" : "AaA_zblDSxSkm2mDLpxpkg",
"version" : {
"created" : "7030099"
}
}
}
}
}
elasticsearch.yml
cluster.name: cluster
node.name: node1
path.data:
- /data
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: true
network.host: ip
discovery.seed_providers: file
cluster.initial_master_nodes: [ "node1", "node2", "node3" ]
action.destructive_requires_name: true
xpack.monitoring.enabled: false
xpack.monitoring.collection.enabled: false
xpack.security.enabled: true
xpack.security.authc.realms.file.csc_file.order: 0
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: "key.key"
xpack.security.http.ssl.certificate: "cert.crt"
xpack.security.http.ssl.verification_mode: "certificate"
xpack.security.http.ssl.certificate_authorities: [ "ca.crt" ]
xpack.security.http.ssl.supported_protocols: TLSv1.2
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.key: "key.key"
xpack.security.transport.ssl.certificate: "cert.crt"
xpack.security.transport.ssl.verification_mode: "certificate"
xpack.security.transport.ssl.certificate_authorities: [ "ca.crt" ]
xpack.security.transport.ssl.supported_protocols: TLSv1.2