Hello Team,
I am trying to push approx 10000 data by converting the pandas data frame to elk.
But it fails to push all, some times it showing following error
failed to establish a new connection errno 111 connection refused
failed to establish a new connection errno 110 connection timeout
#converting to dataframe
df = DataFrame(result,columns=["summarized_test_data_id","package_id","panel_id"]
#formatting the data
def generator(df1):
for c, line in enumerate(df1):
yield {
'_index':''index_name',
#'_type':''index_name1',
'_type':'_doc',
'_id':line.get('summarized_test_data_id'),
'_source':{
'summarized_test_data_id':line.get('summarized_test_data_id',['No Data']),
'package_id':line.get('package_id',['No Data']),
'panel_id':line.get('panel_id',['No Data']),
'test_id':line.get('test_id',['No Data']),
'loinc_nm':line.get('loinc_nm',['No Data']),
}
}
res=helpers.bulk(es,generator(df1),refresh=True,request_timeout=3600)
it seems the following error,But in actual it pushed only 10 records to index
BulkIndexError: ('200 document(s) failed to index.', [{'index': {'_index': 'in-dev-patient-summary-research5', '_type': '_doc', '_id': '15765', 'status': 400, 'error': {'type': 'mapper_parsing_exception', 'reason': 'failed to parse', 'caused_by': {'type': 'json_parse_exception', 'reason': "Non-standard token 'NaN': enable JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS to allow\n at [Source: (org.elasticsearch.common.bytes.AbstractBytesReference$MarkSupportingStreamInputWrapper); line: 1, column: 66]"}},