Unable to parse field in Kibana

Elastic Stack Logstash
1

Hi Team,

I am gathering JSON formats to logstash and then parsing it to elasticsearch.

In that process, there is a key which has array brackets to it. The key gets saved in Elasticsearch as unknown field9Already tried refreshing the Kibana Index patterns). The key is displayed as unknown field in kibana while i try to search.

The Json file I try to parse is below

{
  "_index": "for-sesadasm-staastus-2020.11",
  "_type": "_doc",
  "_id": "qp2J23UBWs_DXkrlIDrk",
  "_version": 1,
  "_score": null,
  "_source": {
    "commit": {
      "parents": [
        {
          "sha": "5e7cf71805d3e6ff611d08883a1bf5fb384c157e",
          "url": "https://URL/api/v3/repos/repo/path/commits/5e7cf71805d3e6ff611d08883a1bf5fb384c157e"
        },
        {
          "sha": "f8851e0e028adfd7b7ed4d9e20f4dd2b190e1d05",
          "url": "https://URL/api/v3/repos/repo/path/commits/f8851e0e028adfd7b7ed4d9e20f4dd2b190e1d05"
        }
      ],
      "sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
      "author": null,
      "url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a",
      "committer": null,
      "commit": {
        "tree": {
          "sha": "0a033fe79515d03d9fa274885cd082580e22f764",
          "url": "https://URL/api/v3/repos/repo/path/git/trees/0a033fe79515d03d9fa274885cd082580e22f764"
        },
        "message": "Merge branch 'releases/test1000_main' into usr/usr1/URL",
        "comment_count": 0,
        "verification": {
          "verified": false,
          "payload": null,
          "reason": "unsigned",
          "signature": null
        },
        "author": {
          "email": "conor@org.com",
          "name": "usr1",
          "date": "2020-11-18T12:17:51Z"
        },
        "committer": {
          "email": "conor@org.com",
          "name": "usr1",
          "date": "2020-11-18T12:17:51Z"
        },
        "url": "https://URL/api/v3/repos/repo/path/git/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
      }
    },
    "id": 17847459,
    "headers": {
      "x_github_easvent": "status",
      "content_length": "10009",
      "http_useasr_agent": "GitHub-Hookshot/d88a271",
      "x_githubas_delivery": "ca3c39bc-29a1-11eb-8476-17f862a87199"
    },
    "context": "conasdasdasdtinuous-intasdasdasdegration/jasdasdenasasdkins/basdranasdasdch",
    "sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
    "enterprise": {
      "website_url": null,
      "created_at": "2019-04-28T18:39:22.000Z",
      "updated_at": "2019-04-28T18:39:22Z",
      "description": null
    },
    "target_url": "target_url",
    "sender": {
      "login": "svc-vmaxghe",
      "id": 2460,
      "site_admin": false,
      "url": "https://URL/api/v3/users/svc-vmaxghe",
      "type": "User"
    },
    "@version": "1",
    "created_at": "2020-11-18T13:27:22+00:00",
    "updated_at": "2020-11-18T13:27:22+00:00",
    "repository": {
      "size": 14471410,
      "has_issues": true,
      "owner": {
        "login": "test",
        "id": 7326,
        "site_admin": false,
        "url": "https://URL/api/v3/users/test",
        "type": "Organization"
      },
      "license": null,
      "watchers_count": 3,
      "pushed_at": "2020-11-18T13:23:36Z",
      "updated_at": "2020-11-18T03:58:02Z",
      "archived": false,
      "forks": 0,
      "disabled": false,
      "has_pages": false,
      "private": true,
      "mirror_url": null,
      "name": "seasasdm",
      "has_projects": true,
      "description": "asdasd",
      "id": 22043,
      "open_issues": 43,
      "homepage": "",
      "url": "https://URL/api/v3/repos/repo/path",
      "created_at": "2019-12-04T19:17:27Z",
      "full_name": "repo/path",
      "fork": false,
      "forks_count": 0,
      "has_downloads": true,
      "language": "Java",
      "default_branch": "trunk",
      "open_issues_count": 43,
      "watchers": 3
    },
    "host": "10.2552.20.223",
    "@timestamp": "2020-11-18T13:27:00.150Z",
    "organization": {
      "login": "test",
      "id": 7326,
      "url": "https://URL/api/v3/orgs/test",
      "description": ""
    },
    "state": "success",
    "type": "json",
    "name": "repo/path",
    "branches": [
      {
        "name": "usr/usr1/URL",
        "protected": false,
        "commit": {
          "sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
          "url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
        }
      }
    ],
    "description": "This commit looks good"
  },
  "fields": {
    "enterprise.created_at": [
      "2019-04-28T18:39:22.000Z"
    ],
    "enterprise.updated_at": [
      "2019-04-28T18:39:22.000Z"
    ],
    "@timestamp": [
      "2020-11-18T13:27:00.150Z"
    ],
    "updated_at": [
      "2020-11-18T13:27:22.000Z"
    ],
    "created_at": [
      "2020-11-18T13:27:22.000Z"
    ],
    "commit.commit.committer.date": [
      "2020-11-18T12:17:51.000Z"
    ],
    "repository.updated_at": [
      "2020-11-18T03:58:02.000Z"
    ],
    "commit.commit.author.date": [
      "2020-11-18T12:17:51.000Z"
    ]
  },
  "sort": [
    1605706020150
  ]
}

When I view it in Kibana as Table. it is showing that the Key "Branches" as Unknown field where as I want to perform certain search on top of it. PFB, the table view from kibana.

branches
{
  "name": "usr/usr1/URL",
  "protected": false,
  "commit": {
    "sha": "a4bb986f56f0903f8eae651d8bc56b377e9a718a",
    "url": "https://URL/api/v3/repos/repo/path/commits/a4bb986f56f0903f8eae651d8bc56b377e9a718a"
  }
}

Is there anyway, i can take the name key under the branches?

Note : I already have other logstash filters in place to filter other fields in this JSON.

Regards
TJ

2

I used the split filter to split this value into branches.name.

This thread can be closed.

The logstash split filter

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.