Reindexing 1.7 to 2.4.4 fails


#1

Hi,
I am using migration plugin to reindex 1.7.1 indices to 2.4.4, so we can migrate to 5.x.
I managed to reindex some indices but most of them are failing. New index gets created but reindex fails. I tried to do it manually, copy>pasting the reindex command from the migration tool (and changing wait_for_completion to true).
Here is an example of one command and the output I get:

curl -XPOST 'http://elasearch03001.pod01:9201/_reindex?wait_for_completion=true' -d '
{
"source": {
"index": "sessions_org4_2017_07_17",
"sort": "_doc",
"size": 1000
},
"dest": {
"index": "sessions_org4_2017_07_17-2.4.4",
"version_type": "external"
}
}
'

What does this output mean "source is missing"? Both indices are present. One 2.4.4 has 0 docs, while the original has 57794 docs.

{"error":{"root_cause":[{"type":"action_request_validation_exception","reason":"Validation Failed: 1: source is missing;2: source is missing;3: source is missing;4: source is missing;5: source is missing;6: source is missing;7: source is missing;8: source is missing;9: source is missing;10: source is missing;11: source is missing;12: source is missing;13: source is missing;14:.....source is missing;724: source is missing;725: source is missing;726: source is missing;727: source is missing;"},"status":400}

Thanks


(Mark Walkom) #2

What does the mapping for the source index look like?


#3

"sessions_org4_2017_07_17":{"mappings":{"_default_":{"_all":{"enabled":false},"_timestamp":{},"dynamic_templates":[{"event":{"mapping":{"index":"not_analyzed","store":false,"type":"integer"},"path_match":"Events.*"}},{"session_attribute":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"SessionAttributes.*"}},{"url_argument":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"screenViews.args.*"}},{"http_header":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"httpHeaders.*"}},{"cookie":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"cookies.*"}},{"dimension":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"match":"Dim*"}},{"generic":{"mapping":{"index":"no","store":false,"type":"{dynamic_type}"},"match":"@&~((Events|SessionAttributes)\\.*)","match_pattern":"regex"}}]},"freetexthit":{"_all":{"enabled":false},"_parent":{"type":"hit"},"_routing":{"required":true},"_source":{"enabled":false},"_timestamp":{},"dynamic_templates":[{"event":{"mapping":{"index":"not_analyzed","store":false,"type":"integer"},"path_match":"Events.*"}},{"session_attribute":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"SessionAttributes.*"}},{"url_argument":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"screenViews.args.*"}},{"http_header":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"httpHeaders.*"}},{"cookie":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"cookies.*"}},{"dimension":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"match":"Dim*"}},{"generic":{"mapping":{"index":"no","store":false,"type":"{dynamic_type}"},"match":"@&~((Events|SessionAttributes)\\.*)","match_pattern":"regex"}}],"properties":{"OrgCode":{"type":"string","analyzer":"custom_lowercase"},"freeText":{"type":"string","analyzer":"standard"}}},"hit":{"_all":{"enabled":false},"_parent":{"type":"session"},"_routing":{"required":true},"_timestamp":{},"dynamic_templates":[{"event":{"mapping":{"index":"not_analyzed","store":false,"type":"integer"},"path_match":"Events.*"}},{"session_attribute":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"SessionAttributes.*"}},{"url_argument":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"screenViews.args.*"}},{"http_header":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"httpHeaders.*"}},{"cookie":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"cookies.*"}},{"dimension":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"match":"Dim*"}},{"generic":{"mapping":{"index":"no","store":false,"type":"{dynamic_type}"},"match":"@&~((Events|SessionAttributes)\\.*)","match_pattern":"regex"}}],"properties":{"CollectorDCName":{"type":"string","index":"no"},"CollectorNodeID":{"type":"string","index":"no"},"Cookies":{"type":"nested","properties":{"Name":{"type":"string","index":"not_analyzed"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"HitDataType":{"type":"string","index":"no"},"HitNumber":{"type":"long"},"HitSource":{"type":"string","index":"no"},"HitTime":{"type":"long"},"HttpHeaders":{"type":"nested","properties":{"Name":{"type":"string","index":"not_analyzed"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"KafkaOffset":{"type":"long"},"KafkaPartition":{"type":"integer"},"KafkaTopic":{"type":"string","index":"not_analyzed"},"OrgCode":{"type":"string","analyzer":"custom_lowercase"},"ProcID":{"type":"integer"},"ProcServerID":{"type":"integer"},"ScreenViewCount":{"type":"integer"},"ScreenViewSources":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"TLTHID":{"type":"string","index":"not_analyzed"},"TLTSID":{"type":"string","analyzer":"custom_lowercase"},"TimeBlock":{"properties":{"DAY_OF_MONTH":{"type":"byte"},"DAY_OF_WEEK":{"type":"byte"},"DAY_OF_YEAR":{"type":"short"},"HOUR_OF_DAY":{"type":"byte"},"MONTH":{"type":"byte"},"QUARTER":{"type":"byte"},"WEEK_OF_YEAR":{"type":"byte"},"YEAR":{"type":"short"}}},"facts":{"type":"nested","properties":{"Dimensions":{"type":"nested","properties":{"ID":{"type":"integer"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"EventID":{"type":"long"},"NumericValue":{"type":"double"},"StepNumber":{"type":"long"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"freeText":{"type":"string","analyzer":"standard"},"screenViews":{"properties":{"arguments":{"type":"nested","properties":{"Name":{"type":"string","index":"not_analyzed"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"fragment":{"type":"string","analyzer":"custom_lowercase"},"name":{"type":"string","analyzer":"custom_url"},"queryString":{"type":"string"}}}}},"session":{"_all":{"enabled":false},"_timestamp":{},"dynamic_templates":[{"event":{"mapping":{"index":"not_analyzed","store":false,"type":"integer"},"path_match":"Events.*"}},


#4

{"session_attribute":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"SessionAttributes.*"}},{"url_argument":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"screenViews.args.*"}},{"http_header":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"httpHeaders.*"}},{"cookie":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"path_match":"cookies.*"}},{"dimension":{"mapping":{"index":"analyzed","store":false,"analyzer":"custom_lowercase","type":"string"},"match":"Dim*"}},{"generic":{"mapping":{"index":"no","store":false,"type":"{dynamic_type}"},"match":"@&~((Events|SessionAttributes)\\.*)","match_pattern":"regex"}}],"properties":{"Active":{"type":"boolean"},"AppKey":{"type":"string","analyzer":"custom_lowercase"},"BrowserName":{"type":"string","analyzer":"custom_lowercase"},"BrowserUserAgent":{"type":"string","analyzer":"custom_lowercase"},"BrowserVersion":{"type":"string","analyzer":"custom_lowercase"},"City":{"type":"string","index":"not_analyzed"},"ClientDuration":{"type":"long"},"CloseEventSource":{"type":"string","index":"no"},"CloseReason":{"type":"string","index":"no"},"CollectorDCName":{"type":"string","index":"no"},"CollectorNodeID":{"type":"string","index":"no"},"Continent":{"type":"string","index":"not_analyzed"},"Country":{"type":"string","index":"not_analyzed"},"CountryCode":{"type":"string","index":"not_analyzed"},"DOMCaptureMissingDCIDs":{"type":"integer"},"Device":{"type":"string","analyzer":"custom_lowercase"},"DeviceManufacturer":{"type":"string","analyzer":"custom_lowercase"},"DeviceMarketingName":{"type":"string","analyzer":"custom_lowercase"},"DeviceModel":{"type":"string","analyzer":"custom_lowercase"},"DeviceTablet":{"type":"boolean"},"DeviceTouchScreen":{"type":"boolean"},"DeviceVendor":{"type":"string","analyzer":"custom_lowercase"},"DeviceYearReleased":{"type":"integer"},"DiscardEventSource":{"type":"string","index":"no"},"DiscardHitNumber":{"type":"integer"},"Discarded":{"type":"boolean"},"DiscardedAnonUBXEvents":{"type":"integer"},"Duration":{"type":"long"},"EventCount":{"type":"integer"},"FactCount":{"type":"integer"},"FirstHitEpoch":{"type":"long"},"FirstScreenView":{"type":"string","analyzer":"custom_lowercase"},"Gestures":{"type":"integer"},"HitContentTypes":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"HitCount":{"type":"integer"},"HitDataTypes":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"HitSources":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"IP":{"type":"string","index":"not_analyzed"},"InitialMobileBatteryLife":{"type":"integer"},"KafkaOffset":{"type":"long"},"KafkaPartition":{"type":"integer"},"KafkaTopic":{"type":"string","index":"not_analyzed"},"KeyboardUses":{"type":"integer"},"Language":{"type":"string","analyzer":"custom_lowercase"},"LastHitEpoch":{"type":"long"},"LastKafkaOffset":{"type":"long"},"LastScreenView":{"type":"string","analyzer":"custom_lowercase"},"MobileAppName":{"type":"string","analyzer":"custom_lowercase"},"MobileAppVersion":{"type":"string","analyzer":"custom_lowercase"},"MobileBatteryLife":{"type":"integer"},"MobileBatteryLifeDelta":{"type":"integer"},"MobileCarrier":{"type":"string","analyzer":"custom_lowercase"},"MobileDevice":{"type":"boolean"},"MobileExceptions":{"type":"integer"},"MobileNetworkReachability":{"type":"string","analyzer":"custom_lowercase"},"MobilePhone":{"type":"boolean"},"NetworkScreenViewContentTypes":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"OS":{"type":"string","analyzer":"custom_lowercase"},"OSVersion":{"type":"string","analyzer":"custom_lowercase"},"OrgCode":{"type":"string","analyzer":"custom_lowercase"},"OrientationChanges":{"type":"integer"},"OrientationDominant":{"type":"string","analyzer":"custom_lowercase"},"Platform":{"type":"string","analyzer":"custom_lowercase"},"PostalCode":{"type":"string","index":"not_analyzed"},"ProcID":{"type":"integer"},"ProcServerID":{"type":"integer"},"SDKMissingFirstSerialNumbers":{"type":"integer"},"SDKMissingSerialNumbers":{"type":"integer"},"ScreenViewCount":{"type":"integer"},"ScreenViewSources":{"properties":{"Count":{"type":"integer"},"Name":{"type":"string","index":"no"}}},"Size":{"type":"long"},"State":{"type":"string","index":"not_analyzed"},"StruggleScore":{"type":"double"},"TLTSID":{"type":"string","analyzer":"custom_lowercase"},"TealeafSDKType":{"type":"string","analyzer":"custom_lowercase"},"TealeafSDKVersion":{"type":"string","analyzer":"custom_lowercase"},"UBXEventCount":{"type":"integer"},"UBXIdentities":{"type":"nested","properties":{"Name":{"type":"string","index":"not_analyzed"},"Value":{"type":"string","analyzer":"custom_lowercase"}}},"UnresponsiveGestures":{"type":"integer"},"events":{"properties":{"Count":{"type":"integer"},"ID":{"type":"integer"}}},"sessionAttributes":{"type":"nested","properties":{"Name":{"type":"string","index":"not_analyzed"},"Value":{"type":"string","analyzer":"custom_lowercase"}}}}}}}}


#5

too long to place it in one post, had to break it in two.


(Mark Walkom) #6

Please format it with code tags :slight_smile:


#7

Sorry about that, it looks better now.


(Christian Dahlqvist) #8

It is still hard to read, but there seems to be something in your index that has _source disabled, which will prevent it from being reindexed.


#9

Yes, I see it too now, freetexthit field.
Thank you Christian.


(system) #10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.