I've just tried this on a new document (with the correct IP):
{
"_index": ".ds-logs-aws-cloudtrail-2022.03.21-000001",
"_id": "pRUjsX8Bqw_w_LoTlr8F",
"_version": 1,
"_score": 1,
"_source": {
"agent": {
"type": "platform-logging"
},
"log": {
"file": {
"path": "AWSLogs/*******/CloudTrail/us-east-1/2022/03/22/*******_CloudTrail_us-east-1_20220322T1015Z_c2KfftAUDHzFHMVt.json.gz"
},
"level": "info",
"logger": "cloudtrail"
},
"error": {},
"cloud": {
"provider": "aws",
"service": {
"name": "iam.amazonaws.com"
},
"region": "us-east-1",
"account": {
"id": "********"
}
},
"@timestamp": "2022-03-22T10:14:26Z",
"ecs": {
"version": "8.0.0"
},
"data_stream": {
"namespace": "default",
"type": "logs",
"dataset": "aws.cloudtrail"
},
"client": {
"ip": "0.0.0.0"
},
"tls": {},
"api": {},
"event": {
"original": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"*******:abailey\",\"arn\":\"arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey\",\"accountId\":\"*******\",\"accessKeyId\":\"*******\",\"sessionContext\":{\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"*******\",\"arn\":\"arn:aws:iam::*******:role/aws-reserved/sso.amazonaws.com/eu-west-1/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda\",\"accountId\":\"787743944430\",\"userName\":\"AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda\"},\"webIdFederationData\":{},\"attributes\":{\"creationDate\":\"2022-03-22T10:13:55Z\",\"mfaAuthenticated\":\"false\"}}},\"eventTime\":\"2022-03-22T10:14:26Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"AWS Internal\",\"userAgent\":\"AWS Internal\",\"requestParameters\":{\"groupName\":\"test-alert-group\"},\"responseElements\":{\"group\":{\"path\":\"/\",\"groupName\":\"test-alert-group\",\"groupId\":\"*******\",\"arn\":\"arn:aws:iam::*******:group/test-alert-group\",\"createDate\":\"Mar 22, 2022 10:14:26 AM\"}},\"requestID\":\"37a01acd-a715-41bf-8dcd-786f57a8f8e1\",\"eventID\":\"82683467-3638-40b1-bab3-2d95ab4707af\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"*******\",\"eventCategory\":\"Management\",\"sessionCredentialFromConsole\":\"true\"}",
"kind": "event",
"module": "aws",
"type": [
"info"
],
"version": "1.08",
"ingested": "2022-03-22T10:19:33.765077006Z",
"provider": "iam.amazonaws.com",
"management": true,
"read_only": false,
"action": "CreateGroup",
"id": "82683467-3638-40b1-bab3-2d95ab4707af",
"category": [
"iam"
],
"dataset": "aws.cloudtrail",
"outcome": "success"
},
"user": {
"access_key_id": "*******",
"session_issuer": {},
"account_id": "********",
"session_context": {},
"id": "arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey",
"type": "AssumedRole",
"arn": "arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey",
"principal_id": "*******:abailey"
}
},
"fields": {
"event.category": [
"iam"
],
"user.principal_id": [
"*******:abailey"
],
"log.logger": [
"cloudtrail"
],
"event.version": [
1.08
],
"agent.type": [
"platform-logging"
],
"event.module": [
"aws"
],
"log.level": [
"info"
],
"user.account_id": [
"********"
],
"event.kind": [
"event"
],
"event.management": [
true
],
"user.access_key_id": [
"********"
],
"event.outcome": [
"success"
],
"agent.type.keyword": [
"platform-logging"
],
"event.original": [
"{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"*******:abailey\",\"arn\":\"arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey\",\"accountId\":\"*******\",\"accessKeyId\":\"*******\",\"sessionContext\":{\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"*******\",\"arn\":\"arn:aws:iam::*******:role/aws-reserved/sso.amazonaws.com/eu-west-1/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda\",\"accountId\":\"*******\",\"userName\":\"AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda\"},\"webIdFederationData\":{},\"attributes\":{\"creationDate\":\"2022-03-22T10:13:55Z\",\"mfaAuthenticated\":\"false\"}}},\"eventTime\":\"2022-03-22T10:14:26Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"AWS Internal\",\"userAgent\":\"AWS Internal\",\"requestParameters\":{\"groupName\":\"test-alert-group\"},\"responseElements\":{\"group\":{\"path\":\"/\",\"groupName\":\"test-alert-group\",\"groupId\":\"*******\",\"arn\":\"arn:aws:iam::*******:group/test-alert-group\",\"createDate\":\"Mar 22, 2022 10:14:26 AM\"}},\"requestID\":\"37a01acd-a715-41bf-8dcd-786f57a8f8e1\",\"eventID\":\"82683467-3638-40b1-bab3-2d95ab4707af\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"********\",\"eventCategory\":\"Management\",\"sessionCredentialFromConsole\":\"true\"}"
],
"cloud.region": [
"us-east-1"
],
"event.read_only": [
false
],
"user.id": [
"arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey"
],
"data_stream.namespace": [
"default"
],
"client.ip": [
"0.0.0.0"
],
"data_stream.type": [
"logs"
],
"cloud.provider": [
"aws"
],
"event.provider": [
"iam.amazonaws.com"
],
"event.ingested": [
"2022-03-22T10:19:33.765Z"
],
"event.action": [
"CreateGroup"
],
"@timestamp": [
"2022-03-22T10:14:26.000Z"
],
"cloud.service.name": [
"iam.amazonaws.com"
],
"cloud.account.id": [
"********"
],
"ecs.version": [
"8.0.0"
],
"user.arn": [
"arn:aws:sts::*******:assumed-role/AWSReservedSSO_AWSAdministratorAccess_3b59e66dd6d68dda/abailey"
],
"log.file.path": [
"AWSLogs/*******/CloudTrail/us-east-1/2022/03/22/*******_CloudTrail_us-east-1_20220322T1015Z_c2KfftAUDHzFHMVt.json.gz"
],
"data_stream.dataset": [
"aws.cloudtrail"
],
"event.type": [
"info"
],
"user.type": [
"AssumedRole"
],
"event.id": [
"82683467-3638-40b1-bab3-2d95ab4707af"
],
"event.dataset": [
"aws.cloudtrail"
]
}
}
In a brand new cluster with no changes at all, this works and shows in the preview. In the existing cluster, this does not work with the same document.
We have set up components in the not-working cluster with the ECS field mapping as below:
{
"_routing": {
"required": false
},
"numeric_detection": false,
"dynamic_date_formats": [
"basic_date_time_no_millis",
"date_time_no_millis"
],
"dynamic": true,
"_source": {
"excludes": [],
"includes": [],
"enabled": true
},
"date_detection": true,
"properties": {
"@timestamp": {
"type": "date"
},
"message": {
"type": "text"
},
"labels": {
"type": "keyword"
},
"tags": {
"type": "keyword"
}
}
}
{
"properties": {
"client": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"nat": {
"type": "object",
"properties": {
"port": {
"type": "long"
},
"ip": {
"type": "ip"
}
}
},
"registered_domain": {
"type": "keyword"
},
"address": {
"type": "keyword"
},
"port": {
"type": "long"
},
"top_level_domain": {
"type": "keyword"
},
"bytes": {
"type": "long"
},
"domain": {
"type": "keyword"
},
"ip": {
"type": "ip"
},
"subdomain": {
"type": "keyword"
},
"mac": {
"type": "keyword"
},
"packets": {
"type": "long"
}
}
}
}
}
{
"dynamic_templates": [],
"properties": {
"cloud": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"availability_zone": {
"type": "keyword"
},
"instance": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
},
"id": {
"type": "keyword"
}
}
},
"macine": {
"type": "object",
"properties": {
"type": {
"type": "keyword"
}
}
},
"provider": {
"type": "keyword"
},
"service": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
}
}
},
"project": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
},
"id": {
"type": "keyword"
}
}
},
"region": {
"type": "keyword"
},
"account": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"name": {
"type": "keyword"
},
"id": {
"type": "keyword"
}
}
}
}
}
}
}
{
"dynamic_templates": [],
"properties": {
"data_stream": {
"type": "object",
"properties": {
"namespace": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"dataset": {
"type": "keyword"
}
}
}
}
}
{
"properties": {
"ecs": {
"type": "object",
"properties": {
"version": {
"type": "keyword"
}
}
}
}
}
{
"properties": {
"error": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"code": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"stack_trace": {
"type": "text"
},
"message": {
"type": "text"
},
"type": {
"type": "keyword"
}
}
}
}
}
{
"properties": {
"event": {
"dynamic": true,
"type": "object",
"enabled": true,
"properties": {
"reason": {
"type": "keyword"
},
"code": {
"type": "keyword"
},
"timezone": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"agent_id_status": {
"type": "keyword"
},
"duration": {
"type": "long"
},
"reference": {
"type": "keyword"
},
"ingested": {
"type": "date"
},
"provider": {
"type": "keyword"
},
"action": {
"type": "keyword"
},
"end": {
"type": "date"
},
"id": {
"type": "text"
},
"outcome": {
"type": "keyword"
},
"severity": {
"type": "long"
},
"original": {
"eager_global_ordinals": false,
"norms": false,
"index": false,
"store": false,
"type": "keyword",
"split_queries_on_whitespace": false,
"doc_values": false
},
"risk_score": {
"type": "float"
},
"kind": {
"type": "keyword"
},
"created": {
"type": "date"
},
"module": {
"type": "keyword"
},
"start": {
"type": "date"
},
"url": {
"type": "keyword"
},
"sequence": {
"type": "long"
},
"risk_score_norm": {
"type": "float"
},
"category": {
"type": "keyword"
},
"dataset": {
"type": "keyword"
},
"hash": {
"type": "keyword"
}
}
}
}
}
{
"properties": {
"log": {
"dynamic": true,
"type": "object",
"enabled": true,
"properties": {
"file": {
"dynamic": true,
"type": "object",
"enabled": true,
"properties": {
"path": {
"type": "keyword"
}
}
},
"level": {
"type": "keyword"
},
"logger": {
"type": "keyword"
},
"origin": {
"type": "object",
"properties": {
"file": {
"type": "object",
"properties": {
"line": {
"type": "long"
},
"name": {
"type": "keyword"
}
}
},
"function": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
}
}
},
"syslog": {
"type": "object",
"properties": {
"severity": {
"type": "object",
"properties": {
"code": {
"type": "long"
},
"name": {
"type": "keyword"
}
}
},
"priority": {
"type": "long"
},
"facility": {
"type": "object",
"properties": {
"code": {
"type": "long"
},
"name": {
"type": "keyword"
}
}
}
}
}
}
}
}
}
{
"properties": {
"tls": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"cipher": {
"type": "keyword"
},
"established": {
"type": "boolean"
},
"server": {
"type": "object",
"properties": {
"not_after": {
"type": "date"
},
"ja3s": {
"type": "keyword"
},
"not_before": {
"type": "date"
},
"subject": {
"type": "keyword"
},
"certificate": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"certificate_chain": {
"type": "keyword"
},
"hash": {
"type": "object",
"properties": {
"sha1": {
"type": "keyword"
},
"sha256": {
"type": "keyword"
},
"md5": {
"type": "keyword"
}
}
},
"issuer": {
"type": "keyword"
},
"version_protocol": {
"type": "keyword"
}
}
},
"curve": {
"type": "keyword"
},
"client": {
"type": "object",
"properties": {
"not_after": {
"type": "date"
},
"server_name": {
"type": "keyword"
},
"not_before": {
"type": "date"
},
"subject": {
"type": "keyword"
},
"supported_ciphers": {
"type": "keyword"
},
"certificate": {
"type": "keyword"
},
"ja3": {
"type": "keyword"
},
"certificate_chain": {
"type": "keyword"
},
"hash": {
"type": "object",
"properties": {
"sha1": {
"type": "keyword"
},
"sha256": {
"type": "keyword"
},
"md5": {
"type": "keyword"
}
}
},
"issuer": {
"type": "keyword"
}
}
},
"next_protocol": {
"type": "keyword"
},
"resumed": {
"type": "boolean"
},
"version": {
"type": "keyword"
}
}
}
}
}
{
"_routing": {
"required": false
},
"numeric_detection": false,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"dynamic": true,
"_source": {
"excludes": [],
"includes": [],
"enabled": true
},
"date_detection": true,
"properties": {
"user": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"full_name": {
"type": "keyword"
},
"domain": {
"type": "keyword"
},
"roles": {
"type": "keyword"
},
"name": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"fields": {
"text": {
"type": "text"
}
},
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"id": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"email": {
"type": "keyword"
},
"hash": {
"type": "keyword"
}
}
}
}
}
{
"properties": {
"user_agent": {
"dynamic": false,
"type": "object",
"enabled": true,
"properties": {
"original": {
"eager_global_ordinals": false,
"norms": false,
"index": true,
"store": false,
"type": "keyword",
"index_options": "docs",
"split_queries_on_whitespace": false,
"doc_values": true
},
"name": {
"type": "keyword"
},
"device": {
"type": "object",
"properties": {
"name": {
"type": "keyword"
}
}
},
"version": {
"type": "keyword"
}
}
}
}
}
As far as I can see, these all conform to the ECS schema. Is there a tool that we can use to check is mapping is causing a problem generating the preview?
