I have about 500000 documents of json data which are quite complex. see below for an example. Essentially i'd like to be able to visualise individual elements of objects within arrays but using the nested data type seems to impede that. in the data below, the example of a visualisation id like to do would be to look at the share of the overall awards.value.amount splitting the metric by awards.suppliers.name.
What is the best way to do this? de serializing the data? using jq or logstash? Any help would be very much appreciated!
{
"_index": "sfp-compranet-ocds-compiled-releases",
"_type": "_doc",
"_id": "ocds-7bcrc6-fffc4a2e-3176-4029-a819-9840afa7fc75",
"_version": 1,
"_score": null,
"_source": {
"ocid": "ocds-7bcrc6-fffc4a2e-3176-4029-a819-9840afa7fc75",
"parties": [
{
"address": {
"streetAddress": "Chief Executive's Department - Angus House - Orchardbank Business Park",
"postalCode": "DD8 1AP",
"countryName": "UK"
},
"roles": [
"buyer"
],
"id": "610b7338-3900-45f3-98b4-2a18194a1e0d",
"name": "Angus Council"
},
{
"address": {
"streetAddress": "Michelin Tyre Premises, Baldovie Road",
"postalCode": "DD4 8UQ",
"countryName": "UK"
},
"roles": [
"supplier"
],
"id": "475ed77b-d9f6-4aa5-9b90-1617718ba6af",
"name": "Abermed Ltd"
},
{
"address": {
"streetAddress": "1230 Lansdowne Court, Gloucester Business Park",
"postalCode": "GL3 4AB",
"countryName": "UK"
},
"roles": [
"supplier"
],
"id": "676cdaa4-13f5-48ff-b384-acf4bcb051e7",
"name": "Care First"
},
{
"address": {
"streetAddress": "Panbride House, Panbride",
"postalCode": "DD7 6JR",
"countryName": "UK"
},
"roles": [
"supplier"
],
"id": "1e51ae09-4cd6-4438-9eea-cad381bd80ee",
"name": "Liz McColgan Health Club and Physiotherapy Centre"
}
],
"flags": {
"i171": {
"types": [
"COMPETITION_CONSTRAINED"
],
"value": false,
"rationale": ";Award=85062.97000000000116415321826934814453125 with tender=94282.4400000000023283064365386962890625;Award=72954.41000000000349245965480804443359375 with tender=94282.4400000000023283064365386962890625;Award=202033.8099999999976716935634613037109375 with tender=94282.4400000000023283064365386962890625"
},
"i085": {
"types": [
"COMPETITION_CONSTRAINED"
],
"value": false,
"rationale": ""
},
"i038": {
"types": [
"COMPETITION_CONSTRAINED"
],
"rationale": "Preconditions that are not met: Needs to have tender start date; "
},
"health": {
"types": [
"HEALTH"
],
"rationale": "Preconditions that are not met: Needs to have tender title; "
},
"eligibleStats": [
{
"count": 3,
"type": "COMPETITION_CONSTRAINED"
}
],
"i007": {
"types": [
"COMPETITION_CONSTRAINED"
],
"value": false,
"rationale": "Number of bids: 3"
}
},
"_class": "org.devgateway.ocds.persistence.mongo.Release",
"id": "ocds-7bcrc6-fffc4a2e-3176-4029-a819-9840afa7fc75-2009-01-01T00:00:00.000Z",
"language": "en",
"date": "2009-01-01T00:00:00.000Z",
"tag": [
"compiled"
],
"@timestamp": "2020-04-23T07:29:18.062Z",
"@version": "1",
"bids": {
"details": [
{
"value": {
"currency": "EUR",
"amount": 85062.97
},
"id": "bid-1",
"tenderers": [
{
"id": "475ed77b-d9f6-4aa5-9b90-1617718ba6af",
"name": "Abermed Ltd"
}
]
},
{
"value": {
"currency": "EUR",
"amount": 72954.41
},
"id": "bid-2",
"tenderers": [
{
"id": "676cdaa4-13f5-48ff-b384-acf4bcb051e7",
"name": "Care First"
}
]
},
{
"value": {
"currency": "EUR",
"amount": 202033.81
},
"id": "bid-3",
"tenderers": [
{
"id": "1e51ae09-4cd6-4438-9eea-cad381bd80ee",
"name": "Liz McColgan Health Club and Physiotherapy Centre"
}
]
}
]
},
"country": "GB",
"buyer": {
"additionalIdentifiers": [],
"id": "610b7338-3900-45f3-98b4-2a18194a1e0d",
"name": "Angus Council"
},
"awards": [
{
"date": {
"$date": "2009-04-08T00:00:00.000Z"
},
"suppliers": [
{
"additionalIdentifiers": [],
"roles": [],
"id": "475ed77b-d9f6-4aa5-9b90-1617718ba6af",
"name": "Abermed Ltd"
}
],
"value": {
"currency": "EUR",
"amount": 85062.97
},
"documents": [
{
"url": {
"scheme": "http",
"host": "data.europa.eu",
"port": -1,
"schemeSpecificPart": "//data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"hash": -766312046,
"path": "/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"authority": "data.europa.eu",
"string": "http://data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv"
},
"datePublished": {
"$date": "2009-01-01T00:00:00.000Z"
},
"documentType": "awardNotice",
"id": "document-1"
}
],
"id": "award-1"
},
{
"date": {
"$date": "2009-04-08T00:00:00.000Z"
},
"suppliers": [
{
"additionalIdentifiers": [],
"roles": [],
"id": "676cdaa4-13f5-48ff-b384-acf4bcb051e7",
"name": "Care First"
}
],
"value": {
"currency": "EUR",
"amount": 72954.41
},
"documents": [
{
"url": {
"scheme": "http",
"host": "data.europa.eu",
"port": -1,
"schemeSpecificPart": "//data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"hash": -766312046,
"path": "/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"authority": "data.europa.eu",
"string": "http://data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv"
},
"datePublished": {
"$date": "2009-01-01T00:00:00.000Z"
},
"documentType": "awardNotice",
"id": "document-1"
}
],
"id": "award-2"
},
{
"date": {
"$date": "2009-04-08T00:00:00.000Z"
},
"suppliers": [
{
"additionalIdentifiers": [],
"roles": [],
"id": "1e51ae09-4cd6-4438-9eea-cad381bd80ee",
"name": "Liz McColgan Health Club and Physiotherapy Centre"
}
],
"value": {
"currency": "EUR",
"amount": 202033.81
},
"documents": [
{
"url": {
"scheme": "http",
"host": "data.europa.eu",
"port": -1,
"schemeSpecificPart": "//data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"hash": -766312046,
"path": "/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv",
"authority": "data.europa.eu",
"string": "http://data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CAN_2009.csv"
},
"datePublished": {
"$date": "2009-01-01T00:00:00.000Z"
},
"documentType": "awardNotice",
"id": "document-1"
}
],
"id": "award-3"
}
],
"initiationType": "tender",
"tender": {
"items": [
{
"additionalClassifications": [],
"classification": {
"scheme": "CPV",
"id": "74530000"
},
"id": "item-1"
},
{
"additionalClassifications": [],
"classification": {
"scheme": "CPV",
"id": "85141000"
},
"id": "item-2"
},
{
"additionalClassifications": [],
"classification": {
"scheme": "CPV",
"id": "85142100"
},
"id": "item-3"
},
{
"additionalClassifications": [],
"classification": {
"scheme": "CPV",
"id": "85312300"
},
"id": "item-4"
}
],
"procurementMethod": "open",
"awardCriteriaDetails": "Price: 70\nQuality: 70",
"value": {
"currency": "EUR",
"amount": 94282.44
},
"id": "2009117896",
"tenderPeriod": {
"endDate": {
"$date": "2008-10-24T00:00:00.000Z"
}
},
"submissionMethod": [],
"mainProcurementCategory": "services",
"contractPeriod": {
"durationInDays": 1170
},
"documents": [
{
"url": {
"scheme": "http",
"host": "data.europa.eu",
"port": -1,
"schemeSpecificPart": "//data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CN_2008.csv",
"hash": 0,
"path": "/euodp/repository/ec/dg-grow/mapps/TED_CN_2008.csv",
"authority": "data.europa.eu",
"string": "http://data.europa.eu/euodp/repository/ec/dg-grow/mapps/TED_CN_2008.csv"
},
"datePublished": {
"$date": "2008-01-01T00:00:00.000Z"
},
"documentType": "tenderNotice",
"id": "document-4"
}
],
"awardCriteria": "ratedCriteria",
"tenderers": []
}
}}