Indicator Match Detection Rule Error - indicators.map is not a function

Elastic Stack Kibana
1

I have not had any issues with this rule until just this morning, but when I click into an Indicator Match Detection Rule we have, the page seems to start loading just fine, but then it gives an error:

Error: indicators.map is not a function
renderRow@https://server.contoso.com/40903/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.7.js:35:189406
StatefulRowRenderer/<@https://server.contoso.com/40903/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.7.js:12:169408
useMemo@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:63499
useMemo@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:361:5228
StatefulRowRenderer@https://server.contoso.com/40903/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.7.js:12:168285
ds@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:59332
xa@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:104414
fl@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:90020
ml@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:89943
ol@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:87291
Gr/<@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:45733
__kbnSharedDeps__</t.unstable_runWithPriority@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:398:3462
qr@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:45442
Gr@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:45680
Vr@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:45613
el@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:84080
As@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:390:62991
next@https://server.contoso.com/40903/bundles/plugin/securitySolution/8.0.0/securitySolution.chunk.7.js:12:247056
__kbnSharedDeps__</u</t.prototype.__tryOrUnsub@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:60242
__kbnSharedDeps__</u</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:59387
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</ui</t.prototype.notifyNext@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:123111
__kbnSharedDeps__</a</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:66678
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</u</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:283:167690
__kbnSharedDeps__</d</t.prototype.nextInfiniteTimeWindow@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:298:609233
next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:117543
__kbnSharedDeps__</u</t.prototype.__tryOrUnsub@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:60242
__kbnSharedDeps__</u</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:59387
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</Ei</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:125130
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</d</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58463
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</vi</t.prototype.nextOrComplete@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:124344
__kbnSharedDeps__</vi</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:124219
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</Ei</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:125130
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
__kbnSharedDeps__</it</t.prototype._next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:329:99915
__kbnSharedDeps__</d</t.prototype.next@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:21:58134
d/</<@https://server.contoso.com/40903/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js:298:176126

Has anybody encountered this before, or perhaps have ideas on how to fix it?

Thank you!
Eric

2

Hi @MakoWish ! This error looks to be coming from the Threat Indicator Match event renderer on the Alerts table. To sidestep the error for the moment, you should be able to disable that event renderer on another page and then be able to view those alerts.

In general, the error indicates that your alerts are not in the expected format for the renderer. Would you be able to share the rule configuration, the source (and format) of your CTI data, and also an example alert (or two) that was generated? That would help us to confirm a bug.

3

Hi @RylandHerrick,

Thank you for the quick response!

The Threat Indicator data is from the Filebeat's "ThreatIntel" module, but instead of writing to filebeat-*, we are writing to threatintel-* (just so the index makes sense when you see it).

Here is the [sanitized] rule:

{
  "id": "943f9800-efca-11eb-bd35-f50e8e50941c",
  "name": "Threat Intel IP Address Indicator Match",
  "tags": ["Elastic", "Threat Intel", "Network"],
  "interval": "10m",
  "enabled": true,
  "description": "This rule is triggered when indicators from the Threat Intel Filebeat module has a match against local network observations. ",
  "risk_score": 100,
  "severity": "critical",
  "note": "An IP address matches that of one known to the Security industry as being malicious. These detections should be considered critical in nature, and an investigation should be performed immediately. ",
  "license": "None",
  "output_index": ".siem-signals-infosec",
  "meta": {
    "from": "2m",
    "kibana_siem_app_url": "https://server.contoso.com/s/infosec/app/security"
  },
  "timestamp_override": "event.ingested",
  "author": ["Somebody"],
  "false_positives": ["None"],
  "from": "now-720s",
  "max_signals": 100,
  "risk_score_mapping": [],
  "severity_mapping": [],
  "threat": [{
      "framework": "MITRE ATT&CK",
      "tactic": {
        "id": "TA0011",
        "reference": "https://attack.mitre.org/tactics/TA0011",
        "name": "Command and Control"
      },
      "technique": [{
          "id": "T1102",
          "reference": "https://attack.mitre.org/techniques/T1102",
          "name": "Web Service",
          "subtechnique": []
        }
      ]
    }
  ],
  "to": "now",
  "references": ["https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html"],
  "version": 3,
  "exceptions_list": [{
      "id": "40f5c530-f16d-11eb-bd35-f50e8e50941c",
      "list_id": "0cbf96ff-4a35-4572-a7ee-290693f9144a",
      "type": "detection",
      "namespace_type": "single"
    }
  ],
  "type": "threat_match",
  "language": "kuery",
  "index": ["winlogbeat-*", "packetbeat-*", "suricata-*", "auditbeat-*"],
  "query": "source.ip:* or destination.ip:* or client.ip:* or server.ip:*",
  "filters": [],
  "threat_filters": [],
  "threat_query": "threatintel.indicator.ip:*",
  "threat_mapping": [{
      "entries": [{
          "field": "source.ip",
          "type": "mapping",
          "value": "threatintel.indicator.ip"
        }
      ]
    }, {
      "entries": [{
          "field": "destination.ip",
          "type": "mapping",
          "value": "threatintel.indicator.ip"
        }
      ]
    }, {
      "entries": [{
          "field": "client.ip",
          "type": "mapping",
          "value": "threatintel.indicator.ip"
        }
      ]
    }, {
      "entries": [{
          "field": "server.ip",
          "type": "mapping",
          "value": "threatintel.indicator.ip"
        }
      ]
    }
  ],
  "threat_language": "kuery",
  "threat_index": ["threatintel-*"],
  "threat_indicator_path": "",
  "throttle": "rule",
  "actions": [{
      "action_type_id": ".email",
      "id": "7fc9e600-de9a-11eb-ab0a-51ff65f25624",
      "params": {
        "subject": "CRITICAL - {{rule.name}}",
        "to": ["AlertMe@Contoso.com"],
        "message": "# Detections:  \n{{#context.alerts}}  \n ----  \n  \nObserver Name: {{{host.name}}}  \nObserver Type: {{{observer.type}}}  \nIntel Source: {{{threat.indicator.event.dataset}}}  \nSource IP: [{{{source.ip}}}](https://server.contoso.com/s/infosec/app/security/network/ip/{{{source.ip}}}/source \"Check IP details in Elastic SIEM\")  \nSource Port: {{{source.port}}}  \nDestination IP: [{{{destination.ip}}}](https://server.contoso.com/s/infosec/app/security/network/ip/{{{destination.ip}}}/destination \"Check IP details in Elastic SIEM\")  \nDestination Port: {{{destination.port}}}  \nMatched Field: {{{threat.indicator.matched.field}}}  \nMatched Type: {{{threat.indicator.matched.type}}}  \n  \n ----  \n{{/context.alerts}}  \n# Description:  \n{{context.rule.description}}  \n# Note:  \n{{{context.rule.note}}}  \n# References:  \n{{#context.rule.references}} {{.}} {{/context.rule.references}}  \n{{#context.rule.threat}}  \n# {{framework}}:  \nTactic: [{{tactic.id}} {{tactic.name}}]({{{tactic.reference}}})  \n{{#technique}}  \n    Technique: [{{id}} {{name}}]({{{reference}}})  \n{{#subtechnique}}  \n        Sub-Technique: [{{id}} {{name}}]({{{reference}}})  \n{{/subtechnique}}  \n{{/technique}}  \n{{/context.rule.threat}}  \n# False Positives:  \n{{context.rule.false_positives}}  \n# Query:  \n{{{context.rule.query}}}  \n# View in Kibana:  \n[{{{context.results_link}}}]({{{context.results_link}}} \"View results in Kibana\")  "
      },
      "group": "default"
    }
  ]
}

Give me a few minutes, and I will work on sanitizing one of the alerts.

1 Like
4

Here is one of the detections:

{
  "_id": "97bdfaed588f59973481c4f4ff23124e37df216842690fd34b2126eaeb18b474",
  "_index": ".siem-signals-infosec-000001",
  "_score": "1",
  "_type": "_doc",
  "@timestamp": "2021-08-10T15:44:10.781Z",
  "@version": "1",
  "agent": {
    "ephemeral_id": "134fe4f4-61cd-4c90-9422-23634a8c2d14",
    "hostname": "observer.contoso.com",
    "id": "7b0bd92a-8957-4499-9149-b7df2a36c534",
    "name": "observer.contoso.com",
    "type": "packetbeat",
    "version": "7.14.0"
  },
  "destination": {
    "bytes": "248",
    "geo": {
      "city_name": "redacted",
      "continent_code": "NA",
      "country_code2": "US",
      "country_code3": "US",
      "country_name": "United States",
      "dma_code": "redacted",
      "latitude": "redacted",
      "location": "{\"lon\":redacted,\"lat\":redacted}",
      "longitude": "redacted",
      "postal_code": "redacted",
      "region_code": "CA",
      "region_name": "California",
      "timezone": "America/Los_Angeles"
    },
    "ip": "redacted",
    "packets": "4",
    "port": "52722"
  },
  "ecs": {
    "version": "1.10.0"
  },
  "event": {
    "action": "network_flow",
    "category": "network_traffic,network",
    "dataset": "flow",
    "duration": "2180705752",
    "end": "2021-08-10T15:37:27.796Z",
    "ingested": "2021-08-10T15:37:56.000Z",
    "kind": "signal",
    "start": "2021-08-10T15:37:25.615Z",
    "timezone": "PDT",
    "type": "connection"
  },
  "flow": {
    "final": "false",
    "id": "EAz/////AP//////CAwAAAEKcgRzQL4+b/LNUABVpT4CAAAAAA"
  },
  "host": {
    "hostname": "observer.contoso.com",
    "name": "observer"
  },
  "logstash": {
    "host": "redacted",
    "hostname": "redacted.contoso.com",
    "port": "5045",
    "protocol": "beats",
    "secured": "true"
  },
  "network": {
    "bytes": "4640",
    "community_id": "1:BiuQangapNBQrpOHOYvXPYNOQX0=",
    "packets": "9",
    "transport": "tcp",
    "type": "ipv4"
  },
  "observer": {
    "hostname": "observer.contoso.com",
    "ip": "redacted",
    "mac": "redacted",
    "name": "redacted",
    "type": "Packetbeat"
  },
  "related": {
    "hosts": "redacted",
    "ip": "redacted"
  },
  "signal": {
    "_meta": {
      "version": "35"
    },
    "ancestors": "{\"id\":\"_Iu2MHsBclDt9G_2k2CY\",\"type\":\"event\",\"index\":\"packetbeat-000019\",\"depth\":0}",
    "depth": "1",
    "original_event": {
      "action": "network_flow",
      "category": "network_traffic,network",
      "dataset": "flow",
      "duration": "2180705752",
      "end": "2021-08-10T15:37:27.796Z",
      "ingested": "2021-08-10T15:37:56.000Z",
      "kind": "event",
      "start": "2021-08-10T15:37:25.615Z",
      "timezone": "PDT",
      "type": "connection"
    },
    "original_time": "2021-08-10T15:37:52.104Z",
    "parent": {
      "depth": "0",
      "id": "_Iu2MHsBclDt9G_2k2CY",
      "index": "packetbeat-000019",
      "type": "event"
    },
    "parents": "{\"id\":\"_Iu2MHsBclDt9G_2k2CY\",\"type\":\"event\",\"index\":\"packetbeat-000019\",\"depth\":0}",
    "rule": {
      "actions": "{\"group\":\"default\",\"params\":{\"subject\":\"CRITICAL - {{rule.name}}\",\"to\":[\"redacted@contoso.com\"],\"message\":\"# Detections:  \\n{{#context.alerts}}  \\n ----  \\n  \\nObserver Name: {{{host.name}}}  \\nObserver Type: {{{observer.type}}}  \\nIntel Source: {{{threat.indicator.event.dataset}}}  \\nSource IP: [{{{source.ip}}}](https://server.contoso.com/s/infosec/app/security/network/ip/{{{source.ip}}}/source \\\"Check IP details in Elastic SIEM\\\")  \\nSource Port: {{{source.port}}}  \\nDestination IP: [{{{destination.ip}}}](https://server.contoso.com/s/infosec/app/security/network/ip/{{{destination.ip}}}/destination \\\"Check IP details in Elastic SIEM\\\")  \\nDestination Port: {{{destination.port}}}  \\nMatched Field: {{{threat.indicator.matched.field}}}  \\nMatched Type: {{{threat.indicator.matched.type}}}  \\n  \\n ----  \\n{{/context.alerts}}  \\n# Description:  \\n{{context.rule.description}}  \\n# Note:  \\n{{{context.rule.note}}}  \\n# References:  \\n{{#context.rule.references}} {{.}} {{/context.rule.references}}  \\n{{#context.rule.threat}}  \\n# {{framework}}:  \\nTactic: [{{tactic.id}} {{tactic.name}}]({{{tactic.reference}}})  \\n{{#technique}}  \\n    Technique: [{{id}} {{name}}]({{{reference}}})  \\n{{#subtechnique}}  \\n        Sub-Technique: [{{id}} {{name}}]({{{reference}}})  \\n{{/subtechnique}}  \\n{{/technique}}  \\n{{/context.rule.threat}}  \\n# False Positives:  \\n{{context.rule.false_positives}}  \\n# Query:  \\n{{{context.rule.query}}}  \\n# View in Kibana:  \\n[{{{context.results_link}}}]({{{context.results_link}}} \\\"View results in Kibana\\\")  \"},\"actionTypeId\":\".email\",\"actionRef\":\"action_0\"}",
      "author": "redacted",
      "created_at": "2021-07-28T17:38:11.466Z",
      "created_by": "redacted",
      "description": "This rule is triggered when indicators from the Threat Intel Filebeat module has a match against local network observations. ",
      "enabled": "true",
      "exceptions_list": "{\"id\":\"40f5c530-f16d-11eb-bd35-f50e8e50941c\",\"list_id\":\"0cbf96ff-4a35-4572-a7ee-290693f9144a\",\"type\":\"detection\",\"namespace_type\":\"single\"}",
      "false_positives": "None",
      "filters": "",
      "from": "now-720s",
      "id": "943f9800-efca-11eb-bd35-f50e8e50941c",
      "immutable": "false",
      "index": "winlogbeat-*,packetbeat-*,suricata-*,auditbeat-*",
      "interval": "10m",
      "language": "kuery",
      "license": "None",
      "max_signals": "100",
      "meta": {
        "from": "2m",
        "kibana_siem_app_url": "https://server.contoso.com/s/infosec/app/security"
      },
      "name": "Threat Intel IP Address Indicator Match",
      "note": "An IP address matches that of one known to the Security industry as being malicious. These detections should be considered critical in nature, and an investigation should be performed immediately. ",
      "output_index": ".siem-signals-infosec",
      "query": "source.ip:* or destination.ip:* or client.ip:* or server.ip:*",
      "references": "https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-module-threatintel.html",
      "risk_score": "100",
      "risk_score_mapping": "",
      "rule_id": "4bf96324-992b-4d01-aa94-7d622a47e37b",
      "severity": "critical",
      "severity_mapping": "",
      "tags": "Elastic,Threat Intel,Network",
      "threat": "{\"framework\":\"MITRE ATT&CK\",\"tactic\":{\"id\":\"TA0011\",\"reference\":\"https://attack.mitre.org/tactics/TA0011\",\"name\":\"Command and Control\"},\"technique\":[{\"id\":\"T1102\",\"reference\":\"https://attack.mitre.org/techniques/T1102\",\"name\":\"Web Service\",\"subtechnique\":[]}]}",
      "threat_filters": "",
      "threat_index": "threatintel-*",
      "threat_indicator_path": "",
      "threat_language": "kuery",
      "threat_mapping": "{\"entries\":[{\"field\":\"source.ip\",\"type\":\"mapping\",\"value\":\"threatintel.indicator.ip\"}]},{\"entries\":[{\"field\":\"destination.ip\",\"type\":\"mapping\",\"value\":\"threatintel.indicator.ip\"}]},{\"entries\":[{\"field\":\"client.ip\",\"type\":\"mapping\",\"value\":\"threatintel.indicator.ip\"}]},{\"entries\":[{\"field\":\"server.ip\",\"type\":\"mapping\",\"value\":\"threatintel.indicator.ip\"}]}",
      "threat_query": "threatintel.indicator.ip:*",
      "timestamp_override": "event.ingested",
      "to": "now",
      "type": "threat_match",
      "updated_at": "2021-08-10T15:34:09.907Z",
      "updated_by": "redacted",
      "version": "3"
    },
    "status": "open"
  },
  "source": {
    "bytes": "4392",
    "geo": {
      "continent_code": "EU",
      "country_code2": "DE",
      "country_code3": "DE",
      "country_name": "Germany",
      "ip": "redacted",
      "latitude": "redacted",
      "location": "{\"lon\":redacted,\"lat\":redacted}",
      "longitude": "redacted",
      "timezone": "Europe/Berlin"
    },
    "ip": "redacted",
    "packets": "5",
    "port": "80"
  },
  "tags": "beats_input_raw_event",
  "threat": {
    "indicator": "{\"ip\":\"redacted\",\"type\":\"ipv4-addr\",\"event\":{\"ingested\":\"2021-07-28T09:33:14.274901473Z\",\"created\":\"2021-07-28T09:33:10.798Z\",\"timezone\":\"PDT\",\"kind\":\"enrichment\",\"module\":\"threatintel\",\"category\":\"threat\",\"type\":\"indicator\",\"dataset\":\"threatintel.otx\"},\"matched\":{\"atomic\":\"64.190.62.111\",\"field\":\"source.ip\",\"id\":\"GmB27HoBNgo4EQfiAEKj\",\"index\":\"threatintel-000001\",\"type\":\"ipv4-addr\"}},{\"ip\":\"redacted\",\"type\":\"ipv4-addr\",\"event\":{\"ingested\":\"2021-07-28T09:28:14.113714420Z\",\"timezone\":\"PDT\",\"created\":\"2021-07-28T09:28:10.894Z\",\"kind\":\"enrichment\",\"module\":\"threatintel\",\"category\":\"threat\",\"type\":\"indicator\",\"dataset\":\"threatintel.otx\"},\"matched\":{\"atomic\":\"redacted\",\"field\":\"source.ip\",\"id\":\"bRFx7HoBsICQk8HpbJUj\",\"index\":\"threatintel-000001\",\"type\":\"ipv4-addr\"}}"
  },
  "type": "flow"
}
5

Also to note... This rule is based on the Elastic pre-built rule Threat Intel Indicator Match. That rule proved to be a bit too resource-intensive, so I broke it out to only IP address matches.

1 Like
6

@MakoWish very interesting! I was expecting threat.indicator to be absent based on the error, but it's also possible that you just happened to grab a "good" alert. A few more questions/tasks for you:

  1. Have you made any modifications to the .siem-signals index mappings? Can you share e.g. the result of:

    GET .siem-signals-*/_mapping/field/threat.indicator.*

  2. To verify the presence of "bad" alerts, can you try adding the following filter to the UI: Screen Shot 2021-08-10 at 4.27.23 PM

    and then viewing that rule's page? If that works, then that confirms that there are errant alerts that the UI is not expecting.

  3. If the above is true, you should be able to grab any such alerts from Dev Tools with the following query:

    GET .siem-signals-*/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "exists": {
            "field": "threat.indicator"
          }
        }
      ]
    }
  }
}

    of course if you have other rules, you'll want to filter to just that particular indicator match rule with a filter on e.g. signal.rule.name.

This is looking like a data bug, but I'm still not sure what the errant data looks like, nor how you got there! Any other data you can provide would be useful; thanks for your help!

7

@RylandHerrick, yes. To avoid "legacy" warnings, I created a Component-based template for the SIEM signals.

Here is the result of the query you requested:

{
  ".siem-signals-space_name-000001" : {
    "mappings" : { }
  }
}

Here is the component template:

{
  "index_patterns" : [
    ".siem-signals-space_name-*"
  ],
  "template" : {
    "settings" : {
      "index" : {
        "lifecycle" : {
          "name" : ".siem-signals-space_name",
          "rollover_alias" : ".siem-signals-space_name"
        },
        "mapping" : {
          "total_fields" : {
            "limit" : "10000"
          }
        }
      }
    },
    "mappings" : {
      "_meta" : {
        "version" : 35
      },
      "dynamic" : false,
      "properties" : {
        "_score" : {
          "type" : "long"
        }
      }
    },
    "aliases" : { }
  },
  "composed_of" : [
    "@timestamp",
    "@version",
    "document_id",
    "agent",
    "as",
    "client",
    "cloud",
    "code_signature",
    "container",
    "destination",
    "dll",
    "dns",
    "ecs",
    "error",
    "event",
    "file",
    "geo",
    "group",
    "hash",
    "host",
    "http",
    "interface",
    "labels",
    "log",
    "logstash",
    "message",
    "network",
    "observer",
    "organization",
    "os",
    "package",
    "pe",
    "process",
    "registry",
    "related",
    "rule",
    "server",
    "service",
    "signal",
    "source",
    "span",
    "tags",
    "threat",
    "tls",
    "trace",
    "transaction",
    "url",
    "user",
    "user_agent",
    "vlan",
    "vulnerability",
    "windows",
    "winlog"
  ],
  "version" : 35,
  "_meta" : {
    "description" : "SIEM Signals Component-Based Template"
  }
}

And here is the threat component as was copied out of the original .siem-signals* legacy template:

{
  "template" : {
    "mappings" : {
      "properties" : {
        "threat" : {
          "properties" : {
            "framework" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "name" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "description" : {
              "type" : "text"
            },
            "technique" : {
              "properties" : {
                "reference" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                },
                "name" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                },
                "subtechnique" : {
                  "properties" : {
                    "reference" : {
                      "ignore_above" : 1024,
                      "type" : "keyword"
                    },
                    "name" : {
                      "ignore_above" : 1024,
                      "type" : "keyword"
                    },
                    "id" : {
                      "ignore_above" : 1024,
                      "type" : "keyword"
                    }
                  }
                },
                "id" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                }
              }
            },
            "id" : {
              "ignore_above" : 1024,
              "type" : "keyword"
            },
            "tactic" : {
              "properties" : {
                "reference" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                },
                "name" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                },
                "id" : {
                  "ignore_above" : 1024,
                  "type" : "keyword"
                }
              }
            }
          }
        }
      }
    }
  }
}

Unfortunately, I cannot get onto the Rule's page at all. I can get into the Edit page for the rule, but not the rule itself.

I am still getting the emails from the email Action, so thank you for providing the query to find new alerts, but we are still getting them via email, so that is a good thing! :slight_smile:

I appreciate your help in looking into this!

8

@RylandHerrick,

Thank you! You got my head on the right track, and I got this fixed.

It appears I had only based my threat Component Template on the ECS Threat Field Reference, but that is a far cry from the fields actually being used. I went back and grabbed it from the .siem-signals-default legacy template (like I mistakenly said I did above), rolled-over, reindexed to the new index, and deleted the original index. I am now able to access the Rule's page without issue. :smiley:

Thank you, again!
Eric

For reference, here is the new threat Component:

{
  "template": {
    "mappings": {
      "properties": {
        "threat" : {
          "properties": {
            "framework": {
              "ignore_above": 1024,
              "type": "keyword"
            },
            "indicator": {
              "properties": {
                "as": {
                  "properties": {
                    "number": {
                      "type": "long"
                    },
                    "organization": {
                      "properties": {
                        "name": {
                          "fields": {
                            "text": {
                              "norms": false,
                              "type": "text"
                            }
                          },
                          "ignore_above": 1024,
                          "type": "keyword"
                        }
                      }
                    }
                  }
                },
                "confidence": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "dataset": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "description": {
                  "type": "wildcard"
                },
                "domain": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "email": {
                  "properties": {
                    "address": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "event": {
                  "properties": {
                    "action": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "category": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "code": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "created": {
                      "type": "date"
                    },
                    "dataset": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "duration": {
                      "type": "long"
                    },
                    "end": {
                      "type": "date"
                    },
                    "hash": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "id": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "ingested": {
                      "type": "date"
                    },
                    "kind": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "module": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "original": {
                      "doc_values": false,
                      "ignore_above": 1024,
                      "index": false,
                      "type": "keyword"
                    },
                    "outcome": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "provider": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "reason": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "reference": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "risk_score": {
                      "type": "float"
                    },
                    "risk_score_norm": {
                      "type": "float"
                    },
                    "sequence": {
                      "type": "long"
                    },
                    "severity": {
                      "type": "long"
                    },
                    "start": {
                      "type": "date"
                    },
                    "timezone": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "type": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "url": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "first_seen": {
                  "type": "date"
                },
                "geo": {
                  "properties": {
                    "city_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "continent_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "country_iso_code": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "country_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "location": {
                      "type": "geo_point"
                    },
                    "name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "region_iso_code": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "region_name": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "ip": {
                  "type": "ip"
                },
                "last_seen": {
                  "type": "date"
                },
                "marking": {
                  "properties": {
                    "tlp": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "matched": {
                  "properties": {
                    "atomic": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "field": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "type": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                },
                "module": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "port": {
                  "type": "long"
                },
                "provider": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "scanner_stats": {
                  "type": "long"
                },
                "sightings": {
                  "type": "long"
                },
                "type": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              },
              "type": "nested"
            },
            "tactic": {
              "properties": {
                "id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "name": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "reference": {
                  "ignore_above": 1024,
                  "type": "keyword"
                }
              }
            },
            "technique": {
              "properties": {
                "id": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "name": {
                  "fields": {
                    "text": {
                      "norms": false,
                      "type": "text"
                    }
                  },
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "reference": {
                  "ignore_above": 1024,
                  "type": "keyword"
                },
                "subtechnique": {
                  "properties": {
                    "id": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "name": {
                      "fields": {
                        "text": {
                          "norms": false,
                          "type": "text"
                        }
                      },
                      "ignore_above": 1024,
                      "type": "keyword"
                    },
                    "reference": {
                      "ignore_above": 1024,
                      "type": "keyword"
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}
1 Like
9

Wonderful! I'm glad you were able to figure it out. We're planning to migrate to component templates ourselves, so you shouldn't have to maintain those much longer.

2 Likes
10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.