I am trying to load into my AWS ElasticSearch module from my EC2 instance but am not able to load any data into my ES - am very new to this and any help is highly appreciated.
I created the index below in Kibana:
PUT logstash-trades
{
"settings":{
"number_of_shards":1,
"number_of_replicas":1
},
"mappings":{
"tracetrddoc1":{
"properties":{
"cusip":{
"type":"text"
},
"reportingpartyside":{
"type":"text"
},
"tradedate":{
"type":"date"
},
"marketsegment":{
"type":"integer"
},
"sectorid":{
"type":"integer"
}
}
}
}
}
Below is my logstash config file:
input {
file {
path => "/home/ec2-user/es_data/temptrds.csv"
start_position => "beginning"
}
}
filter {
csv {
separator => "|"
columns => [ "issuepriceid", "tradedate", "tradeweek", "trademonth", "tradequarter", "tradeyear", "sequencenumber", "reportingpartyside", "effectivedate", "price", "yield", "spread", "estimatedquantity", "yearstomaturity", "marketspread", "issueid", "cusip", "liquidityscore", "maturity", "issuedate", "paymenttype", "interestratetype", "paymentfrequency", "redemption", "countrycode", "marketsegment", "snprating", "moodyrating", "isocurrencycode", "sectorid", "issueamount", "amountoutstanding", "callable", "putable", "sinkable", "makewhole", "tracecode", "industry", "parent_party_id", "isinvestmentgrade", "productid"]
}
mutate {convert => ["issuepriceid", "integer"]}
mutate {convert => ["sequencenumber", "integer"]}
mutate {convert => ["price", "float"]}
mutate {convert => ["yield", "float"]}
mutate {convert => ["spread", "float"]}
mutate {convert => ["estimatedquantity", "float"]}
mutate {convert => ["yearstomaturity", "integer"]}
mutate {convert => ["marketspread", "float"]}
mutate {convert => ["issueid", "integer"]}
mutate {convert => ["liquidityscore", "integer"]}
mutate {convert => ["marketsegment", "integer"]}
mutate {convert => ["snprating", "integer"]}
mutate {convert => ["moodyrating", "integer"]}
mutate {convert => ["isocurrencycode", "integer"]}
mutate {convert => ["sectorid", "integer"]}
mutate {convert => ["issueamount", "float"]}
mutate {convert => ["amountoutstanding", "float"]}
mutate {convert => ["productid", "integer"]}
}
output {
elasticsearch {
hosts => ["https://vpc-xxx.us-east-1.es.amazonaws.com:443"]
index => "logstash-trades"
}
}
After running the command: sudo bin/logstash -f /home/ec2-user/es_data/logstash2.config --path.settings /etc/logstash
When i check the number of documents in my index by GET /_cat/indices?v - i see 0 docs.