Hello everyone,
I'm currently having issues with one of my jobs with below error:
org.elasticsearch.hadoop.rest.EsHadoopParsingException: org.elasticsearch.hadoop.EsHadoopIllegalStateException: Position for 'targetTableCount.site_xref: ' not found in row; typically this is caused by a mapping inconsistency
There is no fields using comma or dots.
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch-spark-20_2.11</artifactId>
<version>7.10.2</version>
</dependency>
Sample Data from Elasticsearch index:
"_source": {
"ctrlmJobName": "SPK_EPS_STBL_JOIN_ODM_FSL", "@timestamp": "2020-12-18T09:36:32.033Z",
"sourceTableCount": { "product_hierarchy": "37128",
"stbl_so_shortage-odmB": "0", "item_master": "11850009",
"stbl_odm_part_level-odmB": "0", "stbl_fsl": "77853",
"stbl_so_shortage_agg": "0", "site_xref": "53",
"commodity_xref": "515", "report_rundates": "17",
"base_system_lob": "891", "stbl_odm_part_level": "413",
"stbl_odm": "101736" },
"usecase": "stbl", "message": "SPK_EPS_STBL_JOIN_ODM_FSL completed.",
"status": "COMPLETED", "errorStackTrace": "",
"level": "INFO", "targetTableCount": {
"stbl_so_shortage-balanceODM": "0", "stbl_so_shortage": "101167"
}, "job_name": "SPK_EPS_STBL_JOIN_ODM_FSL",
"start_time": 1608282654000, "timestamp": 1608284191995,
"additional_info": null, "appId": "app-20201218031039-0022",
"user": "", "end_time": 1608284191000,
"experience": "eps", "@version": "1",
"number_of_records": 0 }
Schema for this particular index:
root
|-- @timestamp: timestamp (nullable = true)
|-- @version: string (nullable = true)
|-- additional_info: struct (nullable = true)
| |-- additionalProp1: string (nullable = true)
| |-- additionalProp2: string (nullable = true)
| |-- additionalProp3: string (nullable = true)
|-- appId: string (nullable = true)
|-- appUser: string (nullable = true)
|-- appVersion: string (nullable = true)
|-- ctlmJobName: string (nullable = true)
|-- ctrlMJobName: string (nullable = true)
|-- ctrlmJobName: string (nullable = true)
|-- end_time: long (nullable = true)
|-- entity_name: string (nullable = true)
|-- errorStackTrace: string (nullable = true)
|-- experience: string (nullable = true)
|-- job_name: string (nullable = true)
|-- level: string (nullable = true)
|-- message: string (nullable = true)
|-- number_of_records: long (nullable = true)
|-- operation: string (nullable = true)
|-- sourceTableCount: struct (nullable = true)
| |-- Commodity_Xref: string (nullable = true)
| |-- DSC_Metadata: string (nullable = true)
| |-- Gpobi_Stbl_Buyer: string (nullable = true)
| |-- Gpobi_Stbl_Master: string (nullable = true)
| |-- Itm_Buyer_Scheduler: string (nullable = true)
| |-- Itm_Master_Scheduler: string (nullable = true)
| |-- Mstrdata_rdl: struct (nullable = true)
| | |-- w_resource_d_vw: string (nullable = true)
| |-- Site_Xref: string (nullable = true)
| |-- So_Shortage: string (nullable = true)
| |-- Source_currency_count: string (nullable = true)
| |-- additionalProp1: string (nullable = true)
| |-- additionalProp2: string (nullable = true)
| |-- additionalProp3: string (nullable = true)
| |-- base_system_lob: string (nullable = true)
| |-- buyer_master: string (nullable = true)
| |-- cfg_master: string (nullable = true)
| |-- cfg_master-cfg: string (nullable = true)
| |-- commodity_xref: string (nullable = true)
| |-- csr_report: string (nullable = true)
| |-- currency_rate: string (nullable = true)
| |-- dell_fiscal_calendar: : string (nullable = true)
| |-- digsupchain_master: struct (nullable = true)
| | |-- site_xref: string (nullable = true)
| |-- digsupchain_process: struct (nullable = true)
| | |-- stbl_ads_split: string (nullable = true)
| |-- digsupchain_process stbl_part_level: string (nullable = true)
| |-- digsupchain_process stbl_so_shortage: string (nullable = true)
| |-- digsupchain_stage: struct (nullable = true)
| | |-- csr_report: string (nullable = true)
| |-- dsc_fga_product_flat: string (nullable = true)
| |-- dsc_item_comm_stg: string (nullable = true)
| |-- dsc_item_comm_stg-comm: string (nullable = true)
| |-- dsc_metadata: string (nullable = true)
| |-- dsc_metadata: : string (nullable = true)
| |-- dsc_spark_conf: string (nullable = true)
| |-- item_master: string (nullable = true)
| |-- item_mst_stage: string (nullable = true)
| |-- master_scheduler: string (nullable = true)
| |-- part_aging_vw: string (nullable = true)
| |-- part_gsm: string (nullable = true)
| |-- part_search: string (nullable = true)
| |-- product_hierarchy: string (nullable = true)
| |-- report_rundates: string (nullable = true)
| |-- shortage_collab: string (nullable = true)
| |-- site_master: string (nullable = true)
| |-- site_xref: string (nullable = true)
| |-- site_xref: : string (nullable = true)
| |-- site_xref_emc: string (nullable = true)
| |-- stbl_ads: string (nullable = true)
| |-- stbl_ax2012: string (nullable = true)
| |-- stbl_ax2012_pre_stg: string (nullable = true)
| |-- stbl_ax2012_version: string (nullable = true)
| |-- stbl_brh_dct: string (nullable = true)
| |-- stbl_brh_dct_pre_stg: string (nullable = true)
| |-- stbl_discp_asn: string (nullable = true)
| |-- stbl_ffps: string (nullable = true)
| |-- stbl_fsl: string (nullable = true)
| |-- stbl_fulfillment_center: string (nullable = true)
| |-- stbl_holiday_calendar: string (nullable = true)
| |-- stbl_odm: string (nullable = true)
| |-- stbl_odm_part_level: string (nullable = true)
| |-- stbl_odm_part_level-odmB: string (nullable = true)
| |-- stbl_part_level: string (nullable = true)
| |-- stbl_part_level_agg: string (nullable = true)
| |-- stbl_part_level_agg(EMC): string (nullable = true) | |-- stbl_part_level_agg(LDELL): string (nullable = true)
| |-- stbl_part_level_request: string (nullable = true)
| |-- stbl_report_download_request: string (nullable = true)
| |-- stbl_so_shortage: string (nullable = true)
| |-- stbl_so_shortage-odmB: string (nullable = true)
| |-- stbl_so_shortage-part-1: string (nullable = true)
| |-- stbl_so_shortage-part-2: string (nullable = true)
| |-- stbl_so_shortage_agg: string (nullable = true)
| |-- stbl_so_shortage_request: string (nullable = true)
| |-- stbl_summary-ads:: string (nullable = true)
| |-- stbl_summary:: string (nullable = true)
| |-- stbl_summary_reg_prod_comm:: string (nullable = true)
| |-- stbl_summary_region:: string (nullable = true)
| |-- w_resource_d_vw: string (nullable = true)
|-- start_time: long (nullable = true)
|-- status: string (nullable = true)
|-- system: string (nullable = true)
|-- targetTableCount: struct (nullable = true)
| |-- Commodity_Xref: string (nullable = true)
| |-- DSC_Metadata: string (nullable = true)
| |-- Site_Xref: string (nullable = true)
| |-- Source_currency_count: string (nullable = true)
| |-- additionalProp1: string (nullable = true)
| |-- additionalProp2: string (nullable = true)
| |-- additionalProp3: string (nullable = true)
| |-- base_system_lob: string (nullable = true)
| |-- buyer_master: string (nullable = true)
| |-- buyer_master_emc: string (nullable = true)
| |-- csr_report: string (nullable = true)
| |-- currency_rate: string (nullable = true)
| |-- dell_fiscal_calendar: : string (nullable = true)
| |-- digsupchain_master: struct (nullable = true)
| | |-- site_xref: string (nullable = true)
| |-- digsupchain_process: struct (nullable = true)
| | |-- stbl_ads: string (nullable = true)
| | |-- stbl_ads_split: string (nullable = true)
| |-- digsupchain_process stbl_part_level: string (nullable = true)
| |-- digsupchain_process stbl_so_shortage: string (nullable = true)
| |-- digsupchain_stage: struct (nullable = true)
| | |-- csr_report: string (nullable = true)
| |-- dsc_fga_product_flat: string (nullable = true)
| |-- dsc_metadata: string (nullable = true)
| |-- dsc_metadata: : string (nullable = true)
| |-- item_master: string (nullable = true)
| |-- item_master-cfg: string (nullable = true)
| |-- item_master-comm: string (nullable = true)
| |-- item_mst_stage: string (nullable = true)
| |-- item_mst_stage_archive: string (nullable = true)
| |-- master_scheduler_emc: string (nullable = true)
| |-- part_aging_vw: string (nullable = true)
| |-- part_search: string (nullable = true)
| |-- product_hierarchy: string (nullable = true)
| |-- report_rundates: string (nullable = true)
| |-- site_master: string (nullable = true)
| |-- site_xref: string (nullable = true)
| |-- site_xref: : string (nullable = true)
| |-- stbl_ads: string (nullable = true)
| |-- stbl_ax2012: string (nullable = true)
| |-- stbl_ax2012_pre_stg: string (nullable = true)
| |-- stbl_ax2012_version: string (nullable = true)
| |-- stbl_brh_dct: string (nullable = true)
| |-- stbl_customer: string (nullable = true)
| |-- stbl_customer-delete: string (nullable = true)
| |-- stbl_ffps: string (nullable = true)
| |-- stbl_fiscal_calendar : string (nullable = true)
| |-- stbl_fsl: string (nullable = true)
| |-- stbl_fulfillment_center: string (nullable = true)
| |-- stbl_holiday_calendar: string (nullable = true)
| |-- stbl_monthly: : string (nullable = true)
| |-- stbl_monthly_reg: : string (nullable = true)
| |-- stbl_monthly_reg_comm: : string (nullable = true)
| |-- stbl_monthly_reg_prd: : string (nullable = true)
| |-- stbl_monthly_reg_prd_comm: : string (nullable = true)
| |-- stbl_monthly_region: : string (nullable = true)
| |-- stbl_odm: string (nullable = true)
| |-- stbl_part: string (nullable = true)
| |-- stbl_part-delete: string (nullable = true)
| |-- stbl_part_level: string (nullable = true)
| |-- stbl_part_level-globalAge: string (nullable = true)
| |-- stbl_part_level_agg: string (nullable = true)
| |-- stbl_part_level_agg-delete: string (nullable = true)
| |-- stbl_part_level_agg-dell: string (nullable = true)
| |-- stbl_part_level_request: string (nullable = true)
| |-- stbl_part_order_agg: string (nullable = true)
| |-- stbl_part_order_agg-delete: string (nullable = true)
| |-- stbl_report_download_request: string (nullable = true)
| |-- stbl_rootcause: string (nullable = true)
| |-- stbl_rootcause-delete: string (nullable = true)
| |-- stbl_so_shortage: string (nullable = true)
| |-- stbl_so_shortage-balanceODM: string (nullable = true)
| |-- stbl_so_shortage-part: string (nullable = true)
| |-- stbl_so_shortage_agg: string (nullable = true)
| |-- stbl_so_shortage_agg-delete: string (nullable = true)
| |-- stbl_so_shortage_agg-dell: string (nullable = true)
| |-- stbl_summary: string (nullable = true)
| |-- stbl_summary-delete: string (nullable = true)
| |-- stbl_summary_reg_comm: string (nullable = true)
| |-- stbl_summary_reg_comm-delete: string (nullable = true)
| |-- stbl_summary_reg_prod_comm: string (nullable = true)
| |-- stbl_summary_reg_prod_comm-delete: string (nullable = true)
| |-- stbl_summary_reg_prod_comm:: string (nullable = true)
| |-- stbl_summary_reg_product: string (nullable = true)
| |-- stbl_summary_reg_product-delete: string (nullable = true)
| |-- stbl_summary_region: string (nullable = true)
| |-- stbl_summary_region-delete: string (nullable = true)
| |-- stbl_summary_region:: string (nullable = true)
| |-- stbl_waterfall: string (nullable = true)
| |-- stbl_weekly: : string (nullable = true)
| |-- stbl_weekly_reg: : string (nullable = true)
| |-- stbl_weekly_reg_comm: : string (nullable = true)
| |-- stbl_weekly_reg_prd: : string (nullable = true)
| |-- stbl_weekly_reg_prd_comm: : string (nullable = true)
| |-- stbl_weekly_region: : string (nullable = true)
| |-- w_resource_d_vw: string (nullable = true)
|-- timestamp: long (nullable = true)
|-- usecase: string (nullable = true)
|-- user: string (nullable = true)