Parse Generic Nestaed json without splitting

Hello,

I have valid json string which i want to parse as json and want to stored in elastic

{
	"id": "101834545",
	"date": "06-Apr-2016",
	"Gender": "Male",
	"ActiveStatus": "Active",
	"NameDetails": {
		"Name": [{
			"NameType": "Primary Name",
			"NameValue": {
				"FirstName": "xxx",
				"Surname": "x"
			}
		}]
	},
	"Descriptions": {
		"Description": [{
			"Description1": "1"
		}, {
			"Description1": "3",
			"Description2": "10"
		}]
	},
	"RoleDetail": {
		"Roles": [{
			"RoleType": "Primary Occupation",
			"OccTitle": {
				"OccCat": "16",
				"_text": "Deceased"
			}
		}, {
			"RoleType": "Previous Roles",
			"OccTitle": [{
				"SinceMonth": "Mar",
				"SinceYear": "2005",
				"ToYear": "2005",
				"OccCat": "16"
			}, {
				"SinceDay": "22",
				"SinceMonth": "Oct",
				"SinceYear": "1993",
				"ToDay": "15",
				"ToMonth": "Mar",
				"ToYear": "2003",
				"OccCat": "1"
			}, {
				"SinceDay": "08",
				"SinceMonth": "Dec",
				"SinceYear": "1976",
				"ToDay": "14",
				"ToMonth": "Jul",
				"ToYear": "1978",
				"OccCat": "1"
			}, {
				"SinceMonth": "Nov",
				"SinceYear": "2010",
				"ToMonth": "Jan",
				"ToYear": "2011",
				"OccCat": "16"
			}]
		}]
	},
	"DateDetails": {
		"Date": [{
			"DateType": "Date of Birth",
			"DateValue": {
				"Day": "25",
				"Month": "Jan",
				"Year": "88888"
			}
		}, {
			"DateType": "Deceased Date",
			"DateValue": {
				"Day": "05",
				"Month": "Apr",
				"Year": "2011"
			}
		}, {
			"DateType": "Inactive as of (PEP)",
			"DateValue": {
				"Month": "Jan",
				"Year": "2011"
			}
		}]
	},
	"BirthPlace": {
		"Place": {
			"name": "xxxxxxx"
		}
	},
	"CountryDetails": {
		"Country": [{
			"CountryType": "xxxxxxxxx",
			"CountryValue": {
				"Code": "xxxxxxxxx"
			}
		}, {
			"CountryType": "Resident of",
			"CountryValue": {
				"Code": "xxxxxxxxx"
			}
		}, {
			"CountryType": "Jurisdiction",
			"CountryValue": {
				"Code": "xxxxxxxxxxxx"
			}
		}, {
			"CountryType": "Country of Reported Allegation",
			"CountryValue": {
				"Code": "xxxxxxxxxxxxx"
			}
		}]
	},
	"ProfileNotes": "xxxxxxxxxx"
}

This is my config file

##Input section
input {
  file {
         path => "/Users/Downloads/Factiva_PFA_Feed_XML/AML/*.json"
         type => "json"
         codec => "json"
         start_position => "beginning"
    }
}
##Filter section
filter {
   grok {
      match => ["path","/Users/Downloads/Factiva_PFA_Feed_XML/AML/%{DATA:filename}.json"]
    }
}
filter{
    json{
        source => "message"
    }
}
filter {
  mutate {
    gsub => [ "filename", "\d", "",
          "filename", "_","" ]
  }
  mutate {
    lowercase => ["filename"]
  }
}
##Output section
output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "%{filename}-%{+YYYY.MM.dd}"
    codec => "json"
  }
}

and getting below error .. since the above json is dynamic any filed can be added anytime so i dont know what key added and what type of value added it would be array or object or simple text

I am getting exceptions

[2020-08-26T16:03:47,102][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"%{filename}-2020.08.26", :_type=>"doc", :routing=>nil}, #<LogStash::Event:0x1d7077fb>], :response=>{"index"=>{"_index"=>"%{filename}-2020.08.26", "_type"=>"doc", "_id"=>"fjhUKnQBAFysE7Y4rNfV", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [RoleDetail.Roles.OccTitle] tried to parse field [OccTitle] as object, but found a concrete value"}}}}

file store data in following manner

{"id":"32323232323","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}
{"id":"121212121212","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}
{"id":"54545454545","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}

Is there any generic way to store data as it is in json ?

Try reading this thread. Or search this forum for either "concrete value" or "mapper_parsing_exception". This has been discussed many times.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.