Extract fields from json file in Logstash using json filter

Elastic Stack Logstash
1

I want to extract fields from json file and want to create graphs in kibana

My input.json file consists of similar json elements (Pasting small portion since file is huge):

    "tags": [
      "jdkinstall",
      "class",
      "download",
      "jdkinstall::download",
      "file",
      "default",
      "node"
    ],
    "file": "/etc/puppetlabs/code/environments/aem_prod/modules/jdkinstall/manifests/download.pp",
    "type": "File",
    "title": "/apps/Binariesjdk",
    "line": 7,
    "resource": "6485158671b69e7dsfr0374sadfgfds813ffbaf",
    "environment": "aem_prod",
    "certname": "hostname.com",
    "parameters": {
      "mode": "0750",
      "group": "webadm",
      "owner": "webadm",
      "backup": false,
      "ensure": "directory"
    },
    "exported": false
 }, 
  {
    "tags": [
      "jdkinstall",
      "class",
      "download",
      "jdkinstall::download",
      "default",
      "node"
    ],
    "file": "/etc/puppetlabs/code/environments/aem_prod/modules/jdkinstall/manifests/init.pp",
    "type": "Class",
    "title": "Jdkinstall::Download",
    "line": 48,
    "resource": "024d7ff1df3315b2714621314b671c7daa1b127a",
    "environment": "aem_prod",
    "certname": "usw1aemapppin15.xxxx.xxxx.com",
    "parameters": {
      "before": [
        "Class[Jdkinstall::Installjdk]"
      ],
      "jdk_filename": "jdk-8u144-linux-x64.tar.gz"
    },
    "exported": false
  },
  {
    "tags": [
      "jdkinstall",
      "class",
      "exec",
      "jdkinstall::installjdk",
      "default",
      "node",
      "installjdk"
    ],
    "file": "/etc/puppetlabs/code/environments/aem_prod/modules/jdkinstall/manifests/installjdk.pp",
    "type": "Exec",
    "title": "installJdk",
    "line": 17,
    "resource": "d11a1605ca4503dde92212b2a7451a9ef7bd5bee",
    "environment": "aem_prod",
    "certname": "usw1aemapppin15.xxxx.sssxx.com",
    "parameters": {
      "cwd": "/apps/java",
      "user": "webadm",
      "group": "webadm",
      "command": "/bin/tar -xvzf /apps/Binariesjdk/jdk-8u144-linux-x64.tar.gz",
      "creates": "/apps/java/jdk1.8.0_144",
      "timeout": 0
    },
    "exported": false
  },

My logstash.conf is:

input {
  file {
    codec => multiline
    {
        pattern => "^\s\s{"
        negate => true
        what => previous
    }
    path => "/usr/share/logstash/bin/puppet_metrics/resources.json"
    type => "json"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}

filter     {
    mutate   {
            gsub => [ "message","\[",""]
            gsub => [ "message","\n",""]
        }

    json {
    source => "message"
    }

    mutate {
    add_field => {
    "ENVIRONMENT" => "%{[message][environment]}"
    }
    remove_field => "type"
    remove_field => "@version"
    remove_field => "host"
    remove_field => "path"
    }
}

output {
  elasticsearch {
    hosts => "xxx.xxx.x.xxx:9200"
    manage_template => false
    index => "logstash-%{+YYYY.MM.dd}"
    user => "abhishek"
    password => "2fudge"
  }
    stdout { codec => rubydebug }
}

Rubydebug output I'm getting is as similar to as follows (pasting small portion here since file is huge ):

{
           "tags" => [
        [0] "multiline",
        [1] "_jsonparsefailure"
    ],
    "ENVIRONMENT" => "%{[message][environment]}",
     "@timestamp" => 2018-10-25T08:21:51.640Z,
        "message" => "  {    \"tags\":       \"stickybit\",      \"class\",      \"exec\",      \"oshardening::stickybit\",      \"remove_duplicates_suid\",      \"oshardening\",      \"default\",      \"node\"    ],    \"file\": \"/etc/puppetlabs/code/environments/ithosting_prod/modules/oshardening/manifests/stickybit.pp\",    \"type\": \"Exec\",    \"title\": \"remove_duplicates_suid\",    \"line\": 24,    \"resource\": \"7830d1db770d916fbb67dd7d0779294bbb18c9d0\",    \"environment\": \"ithosting_prod\",    \"certname\": \"mird1ipisdns02.xxxx.xxxxxx.com\",    \"parameters\": {      \"command\": \"/bin/sort -u /root/system.suid.txt -o /root/system.suid.txt\"    },    \"exported\": false  },"
}
{
           "tags" => [
        [0] "multiline",
        [1] "_jsonparsefailure"
    ],
    "ENVIRONMENT" => "%{[message][environment]}",
     "@timestamp" => 2018-10-25T08:21:51.640Z,
        "message" => "  {    \"tags\":       \"package\",      \"sudo::package\",      \"oshardening::sudoers\",      \"class\",      \"oshardening\",      \"sudo\",      \"default\",      \"node\",      \"sudoers\"    ],    \"file\": \"/etc/puppetlabs/code/environments/ithosting_prod/modules/sudo/manifests/package.pp\",    \"type\": \"Package\",    \"title\": \"sudo\",    \"line\": 57,    \"resource\": \"073cb03faaecd0cfe2a37c30b36bea4684f699ab\",    \"environment\": \"ithosting_prod\",    \"certname\": \"mird1ipisdns02.xxxx.xxxxx.com\",    \"parameters\": {      \"ensure\": \"present\"    },    \"exported\": false  },"
}

I want to extract following fields from json input file:

  1. Environment (e.g. environment => aem_prod)
  2. Certname (e.g. certname => usw1aemapppin15.xxxx.sssxx.com)
    3.If type is Class then create a field named class and paste title field there
    (e.g "type": "Class",
    "title": "Jdkinstall::Download", then Class should be Jdkinstall::Download)

I'm stuck since a lot of days please any help Would be appreciated, Thanks in Advance

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.