Hi there,
Due to some custom requirements, I am attempting to ship my WAF logs into Elasticsearch using lambda.
The function is fairly simple Python, as follows:
import re
import requests
from requests_aws4auth import AWS4Auth
region = 'us-east-1' # e.g. us-west-1
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token)
host = '' # the Amazon ES domain, including https://
index = 'awswaf-'
type = 'waflog'
url = host + '/' + index + '/' + type
headers = { "Content-Type": "application/json" }
s3 = boto3.client('s3')
# Lambda execution starts here
def handler(event, context):
for record in event['Records']:
# Get the bucket name and key for the new file
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
##Troubleshooting: Log results in console
#print("Bucket :", bucket)
#print("Key :", key)
# Get, read, and split the file into lines
obj = s3.get_object(Bucket=bucket, Key=key)
body = obj['Body'].read()
lines = body.splitlines()
#Build and send the request
document = lines
r = requests.post(url, auth=awsauth, json=document, headers=headers)
#Log response in console
print("Response: ", r)
This is passing a WAF log which is just JSON (can provide an example if it would help) to the Elasticsearch domain, however I am getting a 400 Bad Request response for it.
I thought that this could be due to an issue with the indexing of the parameters, but I still received the same error after adding the following index:
PUT awswaf-
{
"mappings" : {
"properties" : {
"action" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"formatVersion" : {
"type" : "long"
},
"httpRequest" : {
"properties" : {
"args" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"clientIp" : {
"type" : "keyword",
"fields" : {
"keyword" : {
"type" : "ip"
}
}
},
"country" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"headers" : {
"properties" : {
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"value" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"httpMethod" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"httpVersion" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"uri" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"httpSourceId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"httpSourceName" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"rateBasedRuleList" : {
"properties" : {
"limitKey" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"maxRateAllowed" : {
"type" : "long"
},
"rateBasedRuleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"terminatingRuleId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"terminatingRuleType" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"timestamp" : {
"type" : "date",
"format" : "epoch_millis"
},
"webaclId" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
}
}
Could you please help me to understand why the request is being rejected?
