Okta Module drops a lot of debugContext information from events

Elastic Stack Beats
,
1

I recently noticed that the Okta module (mainly the Elastic pipeline) drops a good amount of data from the debugData field when moving it from json.debugContext to okta.debug_context, and was wondering if this is intended or not.

Looking at the current Okta pipeline, it only moves a few things:

  {
    "rename": {
      "ignore_missing": true,
      "ignore_failure": true,
      "field": "json.debugContext.debugData.deviceFingerprint",
      "target_field": "okta.debug_context.debug_data.device_fingerprint"
    }
  },
  {
    "rename": {
      "field": "json.debugContext.debugData.requestId",
      "target_field": "okta.debug_context.debug_data.request_id",
      "ignore_missing": true,
      "ignore_failure": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.requestUri",
      "target_field": "okta.debug_context.debug_data.request_uri",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "field": "json.debugContext.debugData.threatSuspected",
      "target_field": "okta.debug_context.debug_data.threat_suspected",
      "ignore_missing": true,
      "ignore_failure": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.url",
      "target_field": "okta.debug_context.debug_data.url",
      "ignore_missing": true
    }
  }

However, there can be far more useful fields within the debugContext area. Here is an example from a report suspicious activity event:

"debugContext": {
    "debugData": {
        "requestId": "<random_id_string>",
        "requestUri": "<uri_endpoint>",
        "suspiciousActivityBrowser": "<browser_string>",
        "suspiciousActivityEventCity": "<city_name>",
        "suspiciousActivityEventCountry": "<country_name>",
        "suspiciousActivityEventId": "<event_id>",
        "suspiciousActivityEventIp": "<ip_address>",
        "suspiciousActivityEventLatitude": "<lat>",
        "suspiciousActivityEventLongitude": "<lon>",
        "suspiciousActivityEventState": "<region_name>",
        "suspiciousActivityEventTransactionId": "<id_string>",
        "suspiciousActivityEventType": "system.email.new_device_notification.sent_message",
        "suspiciousActivityOs": "<os_string>",
        "suspiciousActivityTimestamp": "2021-05-08T21:50:16.594Z",
        "url": "<url>"
    }
}

In this case, the suspicious* fields contain information that is not included anywhere else in the event, meaning that this data is being lost during the pipeline process.

(This data is still in event.original but this field can't really be used.)

2

I was able to work around this temporarily by adding the following legacy template, and additional values in the Okta pipeline:

{
  "_doc": {
    "dynamic_templates": [],
    "properties": {
      "okta": {
        "type": "object",
        "properties": {
          "debug_context": {
            "type": "object",
            "properties": {
              "debug_data": {
                "type": "object",
                "properties": {
                  "suspicious_activity_event_type": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_state": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_longitude": {
                    "type": "float"
                  },
                  "suspicious_activity_event_ip": {
                    "type": "ip"
                  },
                  "suspicious_activity_event_latitude": {
                    "type": "float"
                  },
                  "suspicious_activity_event_city": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_browser": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_transaction_id": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_id": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_os": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_country": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_timestamp": {
                    "type": "date"
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityBrowser",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_browser",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventCity",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_city",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventCountry",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_country",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventId",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_id",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventIp",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_ip",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventLatitude",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_latitude",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventLongitude",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_longitude",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventState",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_state",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventTransactionId",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_transaction_id",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventType",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_type",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityOs",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_os",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityTimestamp",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_timestamp",
      "ignore_missing": true
    }
  },
3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.