Hello,
I have valid json string which i want to parse as json and want to stored in elastic
{
"id": "101834545",
"date": "06-Apr-2016",
"Gender": "Male",
"ActiveStatus": "Active",
"NameDetails": {
"Name": [{
"NameType": "Primary Name",
"NameValue": {
"FirstName": "xxx",
"Surname": "x"
}
}]
},
"Descriptions": {
"Description": [{
"Description1": "1"
}, {
"Description1": "3",
"Description2": "10"
}]
},
"RoleDetail": {
"Roles": [{
"RoleType": "Primary Occupation",
"OccTitle": {
"OccCat": "16",
"_text": "Deceased"
}
}, {
"RoleType": "Previous Roles",
"OccTitle": [{
"SinceMonth": "Mar",
"SinceYear": "2005",
"ToYear": "2005",
"OccCat": "16"
}, {
"SinceDay": "22",
"SinceMonth": "Oct",
"SinceYear": "1993",
"ToDay": "15",
"ToMonth": "Mar",
"ToYear": "2003",
"OccCat": "1"
}, {
"SinceDay": "08",
"SinceMonth": "Dec",
"SinceYear": "1976",
"ToDay": "14",
"ToMonth": "Jul",
"ToYear": "1978",
"OccCat": "1"
}, {
"SinceMonth": "Nov",
"SinceYear": "2010",
"ToMonth": "Jan",
"ToYear": "2011",
"OccCat": "16"
}]
}]
},
"DateDetails": {
"Date": [{
"DateType": "Date of Birth",
"DateValue": {
"Day": "25",
"Month": "Jan",
"Year": "88888"
}
}, {
"DateType": "Deceased Date",
"DateValue": {
"Day": "05",
"Month": "Apr",
"Year": "2011"
}
}, {
"DateType": "Inactive as of (PEP)",
"DateValue": {
"Month": "Jan",
"Year": "2011"
}
}]
},
"BirthPlace": {
"Place": {
"name": "xxxxxxx"
}
},
"CountryDetails": {
"Country": [{
"CountryType": "xxxxxxxxx",
"CountryValue": {
"Code": "xxxxxxxxx"
}
}, {
"CountryType": "Resident of",
"CountryValue": {
"Code": "xxxxxxxxx"
}
}, {
"CountryType": "Jurisdiction",
"CountryValue": {
"Code": "xxxxxxxxxxxx"
}
}, {
"CountryType": "Country of Reported Allegation",
"CountryValue": {
"Code": "xxxxxxxxxxxxx"
}
}]
},
"ProfileNotes": "xxxxxxxxxx"
}
This is my config file
##Input section
input {
file {
path => "/Users/Downloads/Factiva_PFA_Feed_XML/AML/*.json"
type => "json"
codec => "json"
start_position => "beginning"
}
}
##Filter section
filter {
grok {
match => ["path","/Users/Downloads/Factiva_PFA_Feed_XML/AML/%{DATA:filename}.json"]
}
}
filter{
json{
source => "message"
}
}
filter {
mutate {
gsub => [ "filename", "\d", "",
"filename", "_","" ]
}
mutate {
lowercase => ["filename"]
}
}
##Output section
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "%{filename}-%{+YYYY.MM.dd}"
codec => "json"
}
}
and getting below error .. since the above json is dynamic any filed can be added anytime so i dont know what key added and what type of value added it would be array or object or simple text
I am getting exceptions
[2020-08-26T16:03:47,102][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"%{filename}-2020.08.26", :_type=>"doc", :routing=>nil}, #<LogStash::Event:0x1d7077fb>], :response=>{"index"=>{"_index"=>"%{filename}-2020.08.26", "_type"=>"doc", "_id"=>"fjhUKnQBAFysE7Y4rNfV", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [RoleDetail.Roles.OccTitle] tried to parse field [OccTitle] as object, but found a concrete value"}}}}
file store data in following manner
{"id":"32323232323","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}
{"id":"121212121212","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}
{"id":"54545454545","action":"add","date":"06-Apr-2016","Gender":"Male","ActiveStatus":"Active","Deceased":"Yes","NameDetails":{"Name":[{"NameType":"Primary Name","NameValue":{"FirstName":"xxxx","Surname":"xxxxxx"}},{"NameType":"Spelling Variation","NameValue":[{"FirstName":"xxxx","Surname":"xxxx"},{"FirstName":"xxxxxx","Surname":"xxxxx"},{"FirstName":"xxxxxxx","Surname":"xxxxxx"}]}]},"Descriptions":{"Description":[{"Description1":"1"},{"Description1":"3","Description2":"10"}]},"RoleDetail":{"Roles":[{"RoleType":"Primary Occupation","OccTitle":{"OccCat":"16","_text":"Deceased"}},{"RoleType":"Previous Roles","OccTitle":[{"SinceMonth":"Mar","SinceYear":"2005","ToYear":"2005","OccCat":"16","_text":"xxxxx"},{"SinceDay":"22","SinceMonth":"Oct","SinceYear":"1993","ToDay":"15","ToMonth":"Mar","ToYear":"2003","OccCat":"1","_text":"xxxxxxxx"},{"SinceDay":"08","SinceMonth":"Dec","SinceYear":"1976","ToDay":"14","ToMonth":"Jul","ToYear":"1978","OccCat":"1","_text":"xxxxxxxx"},{"OccCat":"7","_text":"xxxxxxxx"},{"SinceMonth":"Nov","SinceYear":"2010","ToMonth":"Jan","ToYear":"2011","OccCat":"16","_text":"xxxxxxxx"},{"OccCat":"16","_text":"xxxxxxxx"}]}]},"DateDetails":{"Date":[{"DateType":"Date of Birth","DateValue":{"Day":"25","Month":"Jan","Year":"1937"}},{"DateType":"Deceased Date","DateValue":{"Day":"05","Month":"Apr","Year":"2011"}},{"DateType":"Inactive as of (PEP)","DateValue":{"Month":"Jan","Year":"2011"}}]},"BirthPlace":{"Place":{"name":"xxxxxxxxx"}},"CountryDetails":{"Country":[{"CountryType":"Citizenship","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"Resident of","CountryValue":{"Code":"xxxxxx"}},{"CountryType":"xxxxxx","CountryValue":{"Code":"xxxxxxx"}},{"CountryType":"xxxxxxx","CountryValue":{"Code":"xxxxx"}}]},"ProfileNotes":"xxxxx"}
Is there any generic way to store data as it is in json ?