I need to set flattened type for some fields, e.g., msgParts.urls, msgParts.metadata, filter.modules to avoid the limit of total fields [1000] issue. Below are the current settings that don't work, please advise.
{
"index_templates" : [
{
"name" : "email-proofpoint",
"index_template" : {
"index_patterns" : [
"email-proofpoint"
],
"template" : {
"settings" : {
"index" : {
"lifecycle" : {
"name" : "email"
}
}
},
"mappings" : {
"_routing" : {
"required" : false
},
"numeric_detection" : false,
"dynamic_date_formats" : [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"_source" : {
"excludes" : [ ],
"includes" : [ ],
"enabled" : true
},
"dynamic" : true,
"dynamic_templates" : [ ],
"date_detection" : true,
"properties" : {
"filter" : {
"type" : "flattened"
},
"urls" : {
"type" : "flattened"
},
"msgParts" : {
"type" : "object",
"properties" : {
"metadata" : {
"type" : "flattened"
},
"urls" : {
"type" : "flattened"
}
}
},
"modules" : {
"type" : "flattened"
}
}
}
},
"composed_of" : [ ],
"data_stream" : {
"hidden" : false,
"allow_custom_routing" : false
}
}
}
]
}
Here is a sample log.
{"connection": {"helo": "apc01-sg2-obe.outbound.protection.outlook.com", "protocol": "smtp:smtp", "tls": {"inbound": {"cipher": "ECDHE-RSA-AES256-GCM-SHA384", "version": "TLSv1.2", "cipherBits": 256}}, "ip": "40.107.215.72", "host": "mail-sgaapc01on2072.outbound.protection.outlook.com", "country": "us", "sid": "3g93ubrkc3", "resolveStatus": "ok"}, "msgParts": [{"md5": "92e5522998ca26ff5762bdc3c3b2cbd3", "labeledExt": "txt", "detectedExt": "TXT", "dataBase64": "U0NBTEFSKDB4N2Y2MzFjMDJmYzMwKQ==\n", "urls": [{"url": "http://www.biosjp.com", "src": ["urldefense"], "isRewritten": true}, {"isRewritten": true, "src": ["filter", "urldefense"], "url": "https://www.biosjp.com"}, {"src": ["filter", "urldefense"], "isRewritten": true, "url": "https://www.facebook.com/biosinc"}, {"url": "https://www.linkedin.com/company/bios-inc-", "src": ["filter", "urldefense"], "isRewritten": true}, {"isRewritten": true, "src": ["filter", "urldefense"], "url": "https://www.biosjp.com/images/spacer.gif"}, {"url": "mailto:test@test.com", "src": ["filter"]}], "labeledName": "text.txt", "detectedMime": "text/plain", "metadata": {}, "disposition": "inline", "isCorrupted": false, "isTimedOut": false, "labeledCharset": "iso-2022-jp", "labeledMime": "text/plain", "isVirtual": false, "sizeDecodedBytes": 1898, "isProtected": false, "sha256": "df97df88bceddf1e23ebae6de810a01637afe9b61dca97e69a5c1066738c62fa", "detectedSizeBytes": 1898, "structureId": "0", "textExtracted": "U0NBTEFSKDB4N2Y2M2I3YmMwZDM4KQ==\n", "isDeleted": false, "detectedCharset": "iso-2022-jp", "detectedName": "text.txt", "isArchive": false}, {"labeledCharset": "iso-2022-jp", "labeledExt": "html", "detectedExt": "HTML", "dataBase64": "U0NBTEFSKDB4N2Y2MzFjMDQ4MjU4KQ==\n", "md5": "4ddb7a1ab966cbcb13ba0a15bc623445", "disposition": "inline", "isCorrupted": false, "isTimedOut": false, "metadata": {"generator": "Microsoft Word 15 (filtered medium)"}, "urls": [{"src": ["filter", "urldefense"], "isRewritten": true, "url": "https://www.biosjp.com"}, {"src": ["filter", "urldefense"], "isRewritten": true, "url": "https://www.facebook.com/biosinc"}, {"isRewritten": true, "src": ["filter", "urldefense"], "url": "https://www.linkedin.com/company/bios-inc-"}, {"url": "https://www.biosjp.com/images/spacer.gif", "src": ["filter"]}, {"src": ["filter"], "url": "https://www.biosjp.com/images/linkedin.png"}, {"url": "mailto:test@test.com", "src": ["filter"]}, {"src": ["filter"], "url": "https://www.biosjp.com/images/facebook.png"}], "labeledName": "text.html", "detectedMime": "text/html", "isDeleted": false, "detectedCharset": "iso-2022-jp", "textExtracted": "U0NBTEFSKDB4N2Y2MzM4NjlmYTM4KQ==\n", "isArchive": false, "detectedName": "text.html", "sizeDecodedBytes": 11288, "isVirtual": false, "isProtected": false, "labeledMime": "text/html", "detectedSizeBytes": 11288, "structureId": "0", "sha256": "c72e80da62f29e43541c430e4e108ccaee53a4483203349a0a3d1742be7eba8f"}, {"labeledCharset": "", "md5": "11b602902c392ebdbb8c55fba2f289d5", "detectedExt": "JPG", "dataBase64": "U0NBTEFSKDB4N2Y2MzNhODcyMGE4KQ==\n", "labeledExt": "jpg", "detectedMime": "image/jpeg", "labeledName": "image002.jpg", "urls": [], "disposition": "attached", "isTimedOut": false, "metadata": {}, "isCorrupted": false, "textExtracted": "U0NBTEFSKDB4N2Y2Mzg3YmJkODU4KQ==\n", "detectedCharset": "", "isDeleted": false, "detectedName": "image002.jpg", "isArchive": false, "labeledMime": "image/jpeg", "isVirtual": false, "isProtected": false, "sizeDecodedBytes": 42095, "sha256": "4d0653f016f50d13e68f5bca1d0b4ba8c0e600e7b8b7056b2ccfaf38e7fa9582", "structureId": "0", "detectedSizeBytes": 42095}], "guid": "GwNCjUlEGk4vb6oCknW2WUGKJI6JUhw8", "ts": "2022-05-24T21:10:57.166709-0400", "envelope": {"from": "test@test.com", "rcpts": ["test@test.com"]}, "filter": {"suborgs": {"rcpts": ["0"], "sender": "0"}, "delivered": {"rcpts": ["48c648499f32114993982c2c3fff356f@test.com"]}, "qid": "24P0Ogs5031671", "origGuid": "Cng2B64xZXhsbh8kfWt522sa2R2ih64Q", "isMsgReinjected": true, "routeDirection": "internal", "actions": [{"module": "access", "rule": "messagebanner_new", "action": "annotate-text"}, {"isFinal": true, "action": "continue", "rule": "messagebanner_new", "module": "access"}, {"module": "spf", "rule": "pass", "action": "continue"}, {"action": "add-header", "module": "av", "rule": "inbound_clean"}, {"action": "continue", "module": "av", "rule": "inbound_clean"}, {"action": "add-header", "rule": "inbound_notspam", "module": "spam"}], "modules": {"spf": {"result": "pass", "domain": "biosjp.com"}, "spam": {"langs": ["en", "jp", "pt", "es"], "version": {"engine": "8.12.0-2204290000", "definitions": "main-2205250006"}, "scores": {"engine": 0, "classifiers": {"bulk": 0, "lowpriority": 0, "phish": 0, "mlxlog": 999, "adult": 0, "spam": 0, "impostor": 0, "mlx": 0, "suspect": 0, "malware": 0}, "overall": 0}, "charsets": ["ISO-2022-JP"]}, "urldefense": {"counts": {"total": 8, "rewritten": 8, "unique": 5}, "version": {"engine": "15"}}}, "durationSecs": 0.277186, "verified": {"rcpts": ["test@test.com"]}, "msgSizeBytes": 86599, "routes": ["allow_relay", "default_inbound"], "disposition": "continue"}, "metadata": {"origin": {"data": {"agent": "m0213835.ppops.net", "version": "8.18.4.73", "cid": "kikcustom_hosted"}}, "tzOffsetMins": -240, "tsEpochMs": 1653441057166}, "msg": {"sizeBytes": 91491, "normalizedHeader": {"cc": ["t <t@h.com>, t\t<t@t.com>, t <t@t.com>, t <t@t.com>, Emiko t\t<t@t.com>, t<t@t.com>, t <test@test.com>, t\t<t@t.com>, t\t<t@t.com>, t<t@tp.com>"], "subject": ["test"], "message-id": ["TYZPR03MB5550A7F7C8BE23071F6118BAF0D69@TYZPR03MB5550.apcprd03.prod.outlook.com"], "to": ["t <t@t.com>, \"t@t.com\"\t<t@t.com>"], "from": ["t@t.com>"]}, "header": {"message-id": ["<TYZPR03MB5550A7F7C8BE23071F6118BAF0D69@TYZPR03MB5550.apcprd03.prod.outlook.com>"], "subject": ["test"], "to": ["t <t@t.com>,\r\n \"t@t.com\"\r\n\t<t@t.com>"], "from": ["t <t@t.com>"]}, "lang": "en", "parsedAddresses": {"to": ["t@t.com", "t@t.com"], "cc": ["t@t.com", "t@t.com", "t@t.com", "t@t.com", "t.t@t.com", "t@t.com", "test@test.com", "test@test.com", "test@test.com", "test@test.com"], "from": ["test@test.com"], "fromDisplayNames": ["test"]}}, "final_action": "continue", "final_module": "access", "final_rule": "messagebanner_new", "action_dkimv": [], "action_spf": [{"module": "spf", "rule": "pass", "action": "continue"}], "action_dmarc": []}