Problem Copyping data from one index to other

Elastic Stack Elasticsearch
1

Hi,
I am trying to copy data from source index(sap-trace-app-logs-k8p) to target index k8pindex but I can't see all the data in kibana.I can see only few records.Please let me know what is wrong here.
In my source index I have 23 fields and my target index i have only 8 fields
logstah conf file:

input {
elasticsearch {
hosts => ["10.103.20.64:9200"]
index => "sap-trace-app-logs-k8p"
size => 1000
scroll => "5m"
docinfo => true
}
}
filter {
mutate {
remove_field => ["prd_m1","prd_m4","prd_m6","prd_m2","prd_m5","prd_Timezone","prd_CustomerMessageComponent","prd_LogID","prd_CorrelationID","prd_Location","prd_Session","prd_PassportSession","prd_PassportUserActivityID","prd_PassportConnection","prd_PassportConnectionCounter","prd_Thread"]
}
}
output {
elasticsearch {
hosts => ["10.103.20.64:9200"]
manage_template => false
index => "k8pindex"
document_id => "%{id}"
workers => 1
}
}

Source index schema

{
"sap-trace-app-logs-k8p": {
"aliases": {},
"mappings": {
"trc-prd": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string"
},
"beat": {
"properties": {
"hostname": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"host": {
"type": "string"
},
"input_type": {
"type": "string"
},
"message": {
"type": "string"
},
"offset": {
"type": "long"
},
"prd_Application": {
"type": "string"
},
"prd_Category": {
"type": "string"
},
"prd_CorrelationID": {
"type": "string"
},
"prd_CustomerMessageComponent": {
"type": "string"
},
"prd_DateTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"prd_ErrorMessage": {
"type": "string"
},
"prd_Location": {
"type": "string"
},
"prd_LogID": {
"type": "string"
},
"prd_PassportConnection": {
"type": "string"
},
"prd_PassportConnectionCounter": {
"type": "string"
},
"prd_PassportSession": {
"type": "string"
},
"prd_PassportUserActivityID": {
"type": "string"
},
"prd_RuntimeComponent": {
"type": "string"
},
"prd_Session": {
"type": "string"
},
"prd_Severity": {
"type": "string"
},
"prd_Thread": {
"type": "string"
},
"prd_Timezone": {
"type": "string"
},
"prd_User": {
"type": "string"
},
"prd_m2": {
"type": "string"
},
"prd_m5": {
"type": "string"
},
"prd_trace_m5": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"appl-prd": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string"
},
"beat": {
"properties": {
"hostname": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"host": {
"type": "string"
},
"input_type": {
"type": "string"
},
"message": {
"type": "string"
},
"offset": {
"type": "long"
},
"prd_Application": {
"type": "string"
},
"prd_Category": {
"type": "string"
},
"prd_CorrelationID": {
"type": "string"
},
"prd_CustomerMessageComponent": {
"type": "string"
},
"prd_DateTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"prd_ErrorMessage": {
"type": "string"
},
"prd_Location": {
"type": "string"
},
"prd_LogID": {
"type": "string"
},
"prd_PassportConnection": {
"type": "string"
},
"prd_PassportConnectionCounter": {
"type": "string"
},
"prd_PassportSession": {
"type": "string"
},
"prd_PassportUserActivityID": {
"type": "string"
},
"prd_RuntimeComponent": {
"type": "string"
},
"prd_Session": {
"type": "string"
},
"prd_Severity": {
"type": "string"
},
"prd_Thread": {
"type": "string"
},
"prd_Timezone": {
"type": "string"
},
"prd_User": {
"type": "string"
},
"prd_trace_m5": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "string"
},
"type": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1500534695122",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "bfL8A49AQZq78vHAVy4rlQ",
"version": {
"created": "2030599"
}
}
},
"warmers": {}
}
}

2

You're removing a number of fields there, some of which appear in the original index.

3

yes ,My target index contains only 8 fields.I want to copy 8 fields out of 23 for optimization.If i keep all this 23 field my dashboard will not load.

{
"k8pindex": {
"aliases": {},
"mappings": {
"appl-prd": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string"
},
"beat": {
"properties": {
"hostname": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"host": {
"type": "string"
},
"input_type": {
"type": "string"
},
"message": {
"type": "string"
},
"offset": {
"type": "long"
},
"prd_Application": {
"type": "string"
},
"prd_Category": {
"type": "string"
},
"prd_DateTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"prd_ErrorMessage": {
"type": "string"
},
"prd_RuntimeComponent": {
"type": "string"
},
"prd_Severity": {
"type": "string"
},
"prd_User": {
"type": "string"
},
"prd_trace_m5": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"trc-prd": {
"properties": {
"@timestamp": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"@version": {
"type": "string"
},
"beat": {
"properties": {
"hostname": {
"type": "string"
},
"name": {
"type": "string"
},
"version": {
"type": "string"
}
}
},
"host": {
"type": "string"
},
"input_type": {
"type": "string"
},
"message": {
"type": "string"
},
"offset": {
"type": "long"
},
"prd_Application": {
"type": "string"
},
"prd_Category": {
"type": "string"
},
"prd_DateTime": {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
},
"prd_ErrorMessage": {
"type": "string"
},
"prd_RuntimeComponent": {
"type": "string"
},
"prd_Severity": {
"type": "string"
},
"prd_User": {
"type": "string"
},
"prd_trace_m5": {
"type": "string"
},
"source": {
"type": "string"
},
"tags": {
"type": "string"
},
"type": {
"type": "string"
}
}
}
},
"settings": {
"index": {
"creation_date": "1501069213488",
"k8pindex": {
"settings": {
"index": {
"creation_date": "1499684605213",
"number_of_shards": "5",
"number_of_replicas": "1",
"uuid": "LLKEh-DYRDmSOeEhycfJLw",
"version": {
"created": "2030599"
}
}
},
"mappings": {
"appl-prd": {
"properties": {
"prd_DateTime": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"offset": {
"type": "long"
},
"prd_Severity": {
"type": "string",
"index": "not_analyzed"
},
"input_type": {
"type": "string"
},
"source": {
"type": "string"
},
"message": {
"type": "string"
},
"type": {
"type": "string"
},
"tags": {
"type": "string"
},
"@timestamp": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"prd_RuntimeComponent": {
"type": "string",
"index": "not_analyzed"
},
"prd_ErrorMessage": {
"ignore_above": "10922",
"type": "string",
"index": "not_analyzed"
},
"@version": {
"type": "string"
},
"beat": {
"properties": {
"name": {
"type": "string"
},
"version": {
"type": "string"
},
"hostname": {
"type": "string"
}
}
},
"host": {
"type": "string"
},
"prd_Application": {
"type": "string",
"index": "not_analyzed"
},
"prd_Category": {
"type": "string",
"index": "not_analyzed"
},
"prd_User": {
"type": "string",
"index": "not_analyzed"
}
}
},

    "number_of_shards": "5",
    "number_of_replicas": "1",
    "uuid": "ISEbSo8zRmKI1ulbOwtkIg",
    "version": {
      "created": "2030599"
    }
  }
},
"warmers": {}

}
}

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.