Dynamic data and none index fields

[Rhh (Rolf)]

Hi

I am trying to use ES and I am having trouble to get some fields not indexed. I put data into ES using serilog.

In my setup I have a custom template like this :

{
"index_patterns": [ "tds-" ],
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"default": {
"dynamic_templates": [
{
"numerics_in_fields": {
"path_match": "fields\.[\d+]$",
"match_pattern": "regex",
"mapping": {
"type": "text",
"index": true,
"norms": false
}
}
},
{
"string_fields": {
"match": "",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"index": true,
"norms": false,
"fields": {
"raw": {
"type": "keyword",
"index": true,
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"message": {
"type": "text",
"index": true
},
"subject": {
"type": "text",
"index": true
},
"logger": {
"type": "text",
"index": true
},
"data": {
"type": "object"
}
}
}
}
}
}

In my data I get results like this :

 {
    "_index" : "tds-2020.04.19",
    "_type" : "logevent",
    "_id" : "WtKDkXEBSyGvzhhwIUVi",
    "_score" : 1.0,
    "_source" : {
      "@timestamp" : "2020-04-19T10:16:59.8418722+02:00",
      "level" : "Error",
      "message" : "{\"Message\":\"4 forsøk\",\"p1\":\"heisan\",\"xSystem\":{\"Process\":{\"Id\":20860,\"Name\":\"WindowsFormsApp11\"},\"Thread\":{\"Id\":1,\"Name\":\"trilili\",\"Culture\":\"nb-NO\"}}",
      "fields" : {
        "logger" : "name of application",
        "subject" : "Første forsøk",
        "data" : {
          "p1" : "heisan",
          "system/process/id" : "20860",
          "system/process/name" : "WindowsFormsApp11",
          "system/thread/id" : "1",
          "system/thread/name" : "trilili",
          "system/thread/culture" : "nb-NO"
        }
      }
    }
  },

I would like the fields p1, system/… to be not indexed. How can I achieve that ? I think I need to change my template but everything I tried has so far not worked.

The fields are added by an serilog enritcher with logEvent.AddOrUpdateProperty

[dadoonet (David Pilato)]

You need to set index: false on those fields. See https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index.html

[Rhh (Rolf)]

Hi

Thanks for your feedback. I know about this, but please look at my template

I need fields added dynamically to the data group to be not indexed.

{
"index_patterns": [ "tds-" ],
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"default": {
"dynamic_templates": [
{
"string_fields": {
"match": "",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"index": false,
"norms": false,
"fields": {
"raw": {
"type": "keyword",
"index": false,
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"message": {
"type": "text",
"index": true
},
"subject": {
"type": "text",
"index": true
},
"logger": {
"type": "text",
"index": true
},
"central": {
"type": "nested",
"properties": {
"code": {
"type": "text",
"index": true
},
"name": {
"type": "text"
}
}
},
"vehicle": {
"type": "nested",
"properties": {
"number": {
"type": "text",
"index": true
},
"permit": {
"type": "text"
}
}
},
"driver": {
"type": "nested",
"properties": {
"number": {
"type": "text",
"index": true
},
"name": {
"type": "text"
}
}
},
"booking": {
"type": "nested",
"properties": {
"number": {
"type": "text",
"index": true
},
"text": {
"type": "text"
}
}
},
"data": {
"dynamic": true,
"properties": {}
},
"exceptions": {
"type": "nested",
"properties": {
"depth": {
"type": "text"
},
"HResult": {
"type": "text"
},
"StackTraceString": {
"type": "text"
}
}
}
}
}
}
}

I did think my current mapping would solve this, but no it does not. So how can I cange my mapping to acchive this ?

Regards

[Rhh (Rolf)]

I have also tried to change my template, but no success. Any help on this ?

{
"index_patterns": [ "tds-" ],
"settings": {
"index.refresh_interval": "5s"
},
"mappings": {
"default": {
"dynamic_templates": [
{
{
"dynamic_rule": {
"match": "data",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"index": false,
"norms": false
}
}
},
"standard_rule": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "text",
"index": true,
"norms": false,
"fields": {
"raw": {
"type": "keyword",
"index": true,
"ignore_above": 256
}
}
}
}
}
],
"properties": {
"message": {
"type": "text"
},
"subject": {
"type": "text"
},
"logger": {
"type": "text"
},
"central": {
"type": "nested",
"properties": {
"code": {
"type": "text"
},
"name": {
"type": "text",
"index": false
}
}
},
"vehicle": {
"type": "nested",
"properties": {
"number": {
"type": "text"
},
"permit": {
"type": "text",
"index": false
}
}
},
"driver": {
"type": "nested",
"properties": {
"number": {
"type": "text"
},
"name": {
"type": "text",
"index": false
}
}
},
"booking": {
"type": "nested",
"properties": {
"number": {
"type": "text"
},
"text": {
"type": "text",
"index": false
}
}
},
"data": {
"type": "nested",
"properties": {
},
"exceptions": {
"type": "nested",
"properties": {
"depth": {
"type": "text",
"index": false
},
"HResult": {
"type": "text",
"index": false
},
"StackTraceString": {
"type": "text",
"index": false
}
}
}
}
}
}
}