Possible to grok an Elasticsearch document fields?

[Dom-nik]

Hello!
I'm trying to load some data from an ES index (using Elasticsearch input plugin), parse some of its fields with grok filter and load them back into another ES index. I can't seem to find a way to access a particular field in my input event.

The event doesn't contain tha 'message' field, just a JSON that ES query returned (but not as a string).
Any suggestions on how to approach this?

[niraj_kumar]

What does your current json look like along with your current config.

--
Niraj

[Dom-nik]

Hello!
My JSON looks like this:

{
          "createdByName": "jharrison29",
          "Company": "The Company",
          "assetCategory": "Prestige",
          "SectorCategoryToAsset": "\\Beauty Care,\\Beauty Care\\Skin and Personal Care",
          "Product": [
            {
              "subbrand": null,
              "subsector": null,
              "subsubbrand": "N/A",
              "fpc_id": 80317569,
              "segment": null,
              "category": "Personal Power",
              "sector": null,
              "brand": "Somebrande"
            },
            {
              "subbrand": null,
              "subsector": null,
              "subsubbrand": "N/A",
              "fpc_id": 87448895,
              "segment": null,
              "category": "Pet Care",
              "sector": null,
              "brand": "Somebrand"
            }
          ],
          "asset_id": 16515969,
          "AssetStatus": "\\Final",
          "assetTypeGrouping": "marketing assets",
          "ModifiedDate": "4/8/2016",
          "targetDefinition": "M.Asset",
          "AssetView": "Front",
          "assetgeography": [
            {
              "country": "FR",
              "region": "EMEA"
            },
            {
              "country": "PL",
              "region": "EMEA"
            }
          ],
          "FileExtension": "\\other",
          "@version": "1",
          "assetagency": [
            {
              "agency_name": "\\DMG"
            },
            {
              "agency_name": "\\Proximity"
            }
          ],
          "timestamp": "2016-01-15T17:51:30Z",
          "assetBrand": "Glycerine Non-Kosher Crude",
          "eventType": "DirectDownloadCompleted",
          "InformationSecurityClassification": "\\zzzUnassigned",
          "tags": [],
          "AssetType": "\\Demo\\Animation",
          "@timestamp": "2016-12-15T15:12:08.512Z",
          "ModifiedByName": "pcollins29",
          "isTemplate": "True",
          "CreatedDate": "7/23/2016",
          "usergroup": [
            {
              "group_name": "Power Editors All Facets"
            }
          ],
          "DeploymentLevel": "Product Research",
          "order_id": 82,
          "user": {
            "Employee_Type": "Non-employee",
            "Email": "abanks29@census.gov",
            "HR_Country": "US",
            "Username": "lrivera29@biglobe.ne.jp",
            "HR_Region": "NA",
            "HR_Function": "Marketing"
          },
          "usergroup_group_name": "\\Everyone\\Sector/Category Skin and Personal Care : 378587\\Skin and Personal Care Business Consumers",
          "GeographyToAsset": "\\CANADA GROUP\\CANADA,\\OTHER NA GROUP\\PUERTO RICO,\\US GROUP\\UNITED STATES"
        }
      }

The config file looks like this:

input {
        elasticsearch {
                hosts => "<my_ip>:9200"
                index => "index1"
                query => '{
						  "query":
								{
								  "query_string": {
										"fields": ["_id"],
										"query": "AVjfAZ_kuIdmMA_mhSxU"
										}
								}
						}'
        }
}

filter {
  grok {
        patterns_dir => ["/etc/logstash/patterns"]
        match => { "usergroup_group_name" => [
                "%{CATEGORY:business_consumer_category} Business Consumers",
                "%{CATEGORY:power_editor_category} Power Editors",
                "%{CATEGORY:business_admin_category} Business Admins"]
        }
        add_field => {
                "user_permissions.public_category" => "All Assets"
                "user_permissions.business_consumer_category" => "%{business_consumer_category}"
                "user_permissions.power_editor_category" => "%{power_editor_category}"
                "user_permissions.business_admin_category" => "%{business_admin_category}"
        }
        break_on_match => false
  }
}

output {
    elasticsearch {
        hosts => ["<myip>:9200"]
        index => "grok_test"
        codec => "json"
	}
}

The CATEGORY pattern looks like this:

CATEGORY [a-zA-Z ]+

[Dom-nik]

I got this to work. No need for further debugging. Thanks!