Nested objects, to flatten or split?

Elastic Stack Logstash
1

Hi there, I'm using logstash and Kibana to visualise data stored in json format (about 2gigs worth).

The ELK stack is all 7.4.2

However my understanding is that kibana doesn't support nested arrays of objects, of which my data contains 2 or 3 in each complete json (about 300,000 complete jsons). Below is an example of one of the nested arrays

[{
	"records": [{
		"compiledrelease": {
			"awards": [{
					"id": "award-1",
					"status": "active",
					"date": 1390262400000,
					"value": {
						"amount": 1553321,
						"currency": "GBP"
					},
					"suppliers": [{
						"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
						"name": "Firmus Energy",
						"additionalIdentifiers": []
					}],
					"items": [{
						"id": "item-2",
						"classification": {
							"scheme": "CPV",
							"id": "09123000"
						},
						"additionalClassifications": []
					}],
					"documents": [{
						"id": "document-2",
						"documentType": "awardNotice",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML",
						"datePublished": 1391212800000,
						"language": "en"
					}]
				},
				{
					"id": "award-2",
					"status": "active",
					"value": {
						"amount": 4034818,
						"currency": "GBP"
					},
					"documents": [{
						"id": "document-2",
						"documentType": "awardNotice",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML",
						"datePublished": 1391212800000,
						"language": "en"
					}]
				},
				{
					"id": "award-3",
					"status": "active",
					"date": 1390262400000,
					"value": {
						"amount": 464088,
						"currency": "GBP"
					},
					"suppliers": [{
						"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
						"name": "Firmus Energy",
						"additionalIdentifiers": []
					}],
					"documents": [{
						"id": "document-2",
						"documentType": "awardNotice",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML",
						"datePublished": 1391212800000,
						"language": "en"
					}]
				}
			],
			"language": "en",
			"country": "GB"
		}

	}]
}]

When putting this in the elk stack it just shows the original array, not any key values.

I have tried the following:

  1. Jq to flatten the data which works in differentiating the different nested objects but if possible I'd rather not do that on all the data

  2. The split logstash plug in. This seems to only work on the first object and deletes all subsequent objects so I only get the key values of "award-1"

Is there anyway of splitting it properly so I can use all data in kibana? Would it be easier to categorise each object with award.id and if so how do I do that?

Apologies if anything is not clear, I am very new to this!

Thankyou very much

2

I have managed to acheive almost what i want with this ruby filter:

ruby {
    code => '
        b = event.get("[awards]")
        h = {}
        b.each { |kv|
            h[kv["id"]] = b[0]
        event.set("[awards]", h)
    '
  }

But this repeats the same object ([awards][2]) for each different ID. How can i fix this? (output below)

[{
	"records": [{
		"compiledrelease": {
			"awards": {
				"award-2": {
					"documents": [{
						"documentType": "awardNotice",
						"datePublished": 1391212800000,
						"id": "document-2",
						"language": "en",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML"
					}],
					"suppliers": [{
						"additionalIdentifiers": [],
						"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
						"name": "Firmus Energy"
					}],
					"value": {
						"currency": "GBP",
						"amount": 1553321
					},
					"status": "active",
					"items": [{
						"additionalClassifications": [],
						"classification": {
							"id": "09123000",
							"scheme": "CPV"
						},
						"id": "item-2"
					}],
					"id": "award-1",
					"date": 1390262400000
				},
				"award-1": {
					"documents": [{
						"documentType": "awardNotice",
						"datePublished": 1391212800000,
						"id": "document-2",
						"language": "en",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML"
					}],
					"suppliers": [{
						"additionalIdentifiers": [],
						"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
						"name": "Firmus Energy"
					}],
					"value": {
						"currency": "GBP",
						"amount": 1553321
					},
					"status": "active",
					"items": [{
						"additionalClassifications": [],
						"classification": {
							"id": "09123000",
							"scheme": "CPV"
						},
						"id": "item-2"
					}],
					"id": "award-1",
					"date": 1390262400000
				},
				"award-3": {
					"documents": [{
						"documentType": "awardNotice",
						"datePublished": 1391212800000,
						"id": "document-2",
						"language": "en",
						"url": "http://ted.europa.eu/udl?uri=TED:NOTICE:035856-2014:TEXT:EN:HTML"
					}],
					"suppliers": [{
						"additionalIdentifiers": [],
						"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
						"name": "Firmus Energy"
					}],
					"value": {
						"currency": "GBP",
						"amount": 1553321
					},
					"status": "active",
					"items": [{
						"additionalClassifications": [],
						"classification": {
							"id": "09123000",
							"scheme": "CPV"
						},
						"id": "item-2"
					}],
					"id": "award-1",
					"date": 1390262400000
				}
			},
			"initiationType": "tender",
			"ocid": "ocds-7bcrc6-000070b9-0f03-452e-9e81-1c21f4637899",
			"country": "GB",
			"tag": [
				"compiled"
			],
			"@version": "1",
			"@timestamp": "2019-11-26T11:15:28.873Z",
			"language": "en",
			"buyer": {
				"additionalIdentifiers": [],
				"id": "23b9070a-e598-4d5c-b544-3ddbe477abf1",
				"name": "Queen's University Belfast"
			},
			"parties": [{
					"contactPoint": {
						"telephone": "+44 2890973026",
						"email": "l.taggart@qub.ac.uk",
						"name": "Louise Taggart",
						"url": "http://www.qub.ac.uk"
					},
					"additionalIdentifiers": [],
					"address": {
						"postalCode": "BT7 1NN",
						"streetAddress": "Procurement Office, University Road",
						"countryName": "UK"
					},
					"roles": [
						"buyer"
					],
					"id": "23b9070a-e598-4d5c-b544-3ddbe477abf1",
					"name": "Queen's University Belfast"
				},
				{
					"contactPoint": {
						"telephone": "+44 2894427800",
						"url": "http://www.firmusenergy.co.uk"
					},
					"additionalIdentifiers": [],
					"address": {
						"postalCode": "BT41 4NN",
						"streetAddress": "Killybegs Business Park",
						"countryName": "UK"
					},
					"roles": [
						"supplier"
					],
					"id": "176a1ad2-ca83-4f31-83c9-b1a7a74b2eb8",
					"name": "Firmus Energy"
				}
			]
		}
	}]
}]
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.