Hi Team,
I am gathering JSON formats to logstash and then parsing it to elasticsearch.
In that process, there is a key which has array brackets to it. The key gets saved in Elasticsearch as unknown field9Already tried refreshing the Kibana Index patterns). The key is displayed as unknown field in kibana while i try to search.
The Json file I try to parse is below
{
"_index": "for-sesadasm-staastus-2020.11",
"_type": "_doc",
"_id": "qp2J23UBWs_DXkrlIDrk",
"_version": 1,
"_score": null,
"_source": {
"commit": {
"parents": [
{
"sha": "5e7cf71805d3e6ff611d08883a1bf5fb384c157e",
"url": "https://URL/api/v3/repos/repo/path/commits/5e7cf71805d3e6ff611d08883a1bf5fb384c157e"
},
{
"sha": "f8851e0e028adfd7b7ed4d9e20f4dd2b190e1d05",
"url": "https://URL/api/v3/repos/repo/path/commits/f8851e0e028adfd7b7ed4d9e20f4dd2b190e1d05"
}
],
"sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
"author": null,
"url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a",
"committer": null,
"commit": {
"tree": {
"sha": "0a033fe79515d03d9fa274885cd082580e22f764",
"url": "https://URL/api/v3/repos/repo/path/git/trees/0a033fe79515d03d9fa274885cd082580e22f764"
},
"message": "Merge branch 'releases/test1000_main' into usr/usr1/URL",
"comment_count": 0,
"verification": {
"verified": false,
"payload": null,
"reason": "unsigned",
"signature": null
},
"author": {
"email": "conor@org.com",
"name": "usr1",
"date": "2020-11-18T12:17:51Z"
},
"committer": {
"email": "conor@org.com",
"name": "usr1",
"date": "2020-11-18T12:17:51Z"
},
"url": "https://URL/api/v3/repos/repo/path/git/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
}
},
"id": 17847459,
"headers": {
"x_github_easvent": "status",
"content_length": "10009",
"http_useasr_agent": "GitHub-Hookshot/d88a271",
"x_githubas_delivery": "ca3c39bc-29a1-11eb-8476-17f862a87199"
},
"context": "conasdasdasdtinuous-intasdasdasdegration/jasdasdenasasdkins/basdranasdasdch",
"sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
"enterprise": {
"website_url": null,
"created_at": "2019-04-28T18:39:22.000Z",
"updated_at": "2019-04-28T18:39:22Z",
"description": null
},
"target_url": "target_url",
"sender": {
"login": "svc-vmaxghe",
"id": 2460,
"site_admin": false,
"url": "https://URL/api/v3/users/svc-vmaxghe",
"type": "User"
},
"@version": "1",
"created_at": "2020-11-18T13:27:22+00:00",
"updated_at": "2020-11-18T13:27:22+00:00",
"repository": {
"size": 14471410,
"has_issues": true,
"owner": {
"login": "test",
"id": 7326,
"site_admin": false,
"url": "https://URL/api/v3/users/test",
"type": "Organization"
},
"license": null,
"watchers_count": 3,
"pushed_at": "2020-11-18T13:23:36Z",
"updated_at": "2020-11-18T03:58:02Z",
"archived": false,
"forks": 0,
"disabled": false,
"has_pages": false,
"private": true,
"mirror_url": null,
"name": "seasasdm",
"has_projects": true,
"description": "asdasd",
"id": 22043,
"open_issues": 43,
"homepage": "",
"url": "https://URL/api/v3/repos/repo/path",
"created_at": "2019-12-04T19:17:27Z",
"full_name": "repo/path",
"fork": false,
"forks_count": 0,
"has_downloads": true,
"language": "Java",
"default_branch": "trunk",
"open_issues_count": 43,
"watchers": 3
},
"host": "10.2552.20.223",
"@timestamp": "2020-11-18T13:27:00.150Z",
"organization": {
"login": "test",
"id": 7326,
"url": "https://URL/api/v3/orgs/test",
"description": ""
},
"state": "success",
"type": "json",
"name": "repo/path",
"branches": [
{
"name": "usr/usr1/URL",
"protected": false,
"commit": {
"sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
"url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
}
}
],
"description": "This commit looks good"
},
"fields": {
"enterprise.created_at": [
"2019-04-28T18:39:22.000Z"
],
"enterprise.updated_at": [
"2019-04-28T18:39:22.000Z"
],
"@timestamp": [
"2020-11-18T13:27:00.150Z"
],
"updated_at": [
"2020-11-18T13:27:22.000Z"
],
"created_at": [
"2020-11-18T13:27:22.000Z"
],
"commit.commit.committer.date": [
"2020-11-18T12:17:51.000Z"
],
"repository.updated_at": [
"2020-11-18T03:58:02.000Z"
],
"commit.commit.author.date": [
"2020-11-18T12:17:51.000Z"
]
},
"sort": [
1605706020150
]
}
When I view it in Kibana as Table. it is showing that the Key "Branches" as Unknown field where as I want to perform certain search on top of it. PFB, the table view from kibana.
branches
{
"name": "usr/usr1/URL",
"protected": false,
"commit": {
"sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
"url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
}
}
Is there anyway, i can take the name key under the branches?
Note : I already have other logstash filters in place to filter other fields in this JSON.
Regards
TJ