Hi,
I currently use kibana -> logstash -> elasticsearch to collect data from
CloudTrails. I am running into an issue where all the describe* events are
being loaded to ES properly, but anything instance events (like
RunInstances), is causing parse errors. Just a little background on how I
am getting the logs, after I get them from S3, I parse out all the JSON
into separate events and push them to a log. This way I can have logstash
pick up the log, and then use the JSON filter on these. Here are some
examples:
Works:
{
"userAgent": "Twisted PageGetter",
"sourceIPAddress": "123.456.789.012",
"responseElements": "",
"requestParameters": {
"volumeSet": {
"items": [
{
"volumeId": "vol-123abc"
}
]
},
"filterSet": {}
},
"eventVersion": "1.0",
"userIdentity": {
"type": "IAMUser",
"principalId": "XYXYXYXYXYXYXY",
"arn": "arn:aws:iam::YYYYYYYYYY:user/test123",
"accountId": "YYYYYYYYYY",
"accessKeyId": "XXXXXXXXXXXXXX",
"userName": "test123"
},
"eventTime": "2013-12-09T21:18:11Z",
"eventSource": "ec2.amazonaws.com",
"eventName": "DescribeVolumeStatus",
"awsRegion": "us-east-1"
}
However these do not:
{
"userAgent": "EC2ConsoleBackend",
"sourceIPAddress": "123.456.789.012",
"responseElements": {
"reservationId": "r-c45df43g5",
"ownerId": "XXXXXXXXXXXX",
"groupSet": {
"items": [
{
"groupId": "sg-35cd3456",
"groupName": "testgroup"
}
]
},
"instancesSet": {
"items": [
{
"blockDeviceMapping": {},
"rootDeviceName": "/dev/sda1",
"rootDeviceType": "ebs",
"architecture": "x86_64",
"stateReason": {
"code": "pending",
"message": "pending"
},
"monitoring": {
"state": "disabled"
},
"instanceType": "t1.micro",
"launchTime": 1386623730000,
"virtualizationType": "paravirtual",
"hypervisor": "xen",
"clientToken": "SFfgSVV45672",
"groupSet": {
"items": [
{
"groupId": "sg-36df4532",
"groupName": "TestGroup"
}
]
},
"networkInterfaceSet": {},
"ebsOptimized": false,
"productCodes": {},
"amiLaunchIndex": 0,
"keyName": "testkey",
"instanceState": {
"code": 0,
"name": "pending"
},
"imageId": "ami-83e4defs",
"instanceId": "i-146er456",
"placement": {
"availabilityZone": "us-east-1c",
"tenancy": "default"
},
"kernelId": "aki-testaki"
}
]
}
},
"requestParameters": {
"clientToken": "QDSEFGTEDGHH",
"ebsOptimized": false,
"instancesSet": {
"items": [
{
"imageId": "ami-test",
"minCount": 1,
"maxCount": 1,
"keyName": "testkey"
}
]
},
"groupSet": {
"items": [
{
"groupId": "sg-1212sed23"
}
]
},
"instanceType": "t1.micro",
"blockDeviceMapping": {
"items": [
{
"deviceName": "/dev/sda1",
"ebs": {
"volumeSize": 8,
"deleteOnTermination": true,
"volumeType": "standard"
}
}
]
},
"monitoring": {
"enabled": false
},
"disableApiTermination": false,
"instanceInitiatedShutdownBehavior": "stop"
},
"eventVersion": "1.0",
"userIdentity": {
"type": "IAMUser",
"principalId": "123ABC123ABC",
"arn": "arn:aws:iam::XYXYXYXYXY:user/test123",
"accountId": "YYYYYYYYYYYYYYY",
"accessKeyId": "XXXXXXXXXXXXXX",
"userName": "oweng",
"sessionContext": {
"attributes": {
"mfaAuthenticated": "false",
"creationDate": "2013-12-09T21:14:04Z"
}
}
},
"eventTime": "2013-12-09T21:15:30Z",
"eventSource": "ec2.amazonaws.com",
"eventName": "RunInstances",
"awsRegion": "us-east-1"
}
The error I get is:
org.elasticsearch.index.mapper.MapperParsingException: failed to parse
[responseElements]
at
org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:398)
at
org.elasticsearch.index.mapper.object.ObjectMapper.serializeObject(ObjectMapper.java:519)
at
org.elasticsearch.index.mapper.object.ObjectMapper.parse(ObjectMapper.java:461)
at
org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:513)
at
org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:457)
at
org.elasticsearch.index.shard.service.InternalIndexShard.prepareCreate(InternalIndexShard.java:342)
at
org.elasticsearch.action.bulk.TransportShardBulkAction.shardIndexOperation(TransportShardBulkAction.java:401)
at
org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:155)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary(TransportShardReplicationOperationAction.java:556)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run(TransportShardReplicationOperationAction.java:426)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:722)
Caused by: org.elasticsearch.ElasticSearchIllegalArgumentException: unknown
property [instancesSet]
at
org.elasticsearch.index.mapper.core.StringFieldMapper.parseCreateField(StringFieldMapper.java:280)
at
org.elasticsearch.index.mapper.core.AbstractFieldMapper.parse(AbstractFieldMapper.java:387)
... 12 more
Is this due to the records not having the same amount of fields? Any
guidance would be appreciated. I am totally a newbie with elastic search,
but if this is something I need to grok out fields or something like that,
I think I can handle that.
Thanks again everyone.
Geoff
--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/elasticsearch/2a168123-5f79-4f66-943f-0e10499a455f%40googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.