PLEASE HELP : URGENT REQUEST! I NEED HELP WITH EL
I have just started using elasticsearch to sort the data that comes to my DB (I used to use MySQL earlier)
I get about 30 Million records per-day, and when I try searching for data, the system hangs or crashes even. EL uses up all my memory.
Data scales vertically as if now. And I recently read somewhere that horizontal scaling of data is much better than the vertical approach.
I am lost on how to achieve this using elasticserach.
PLEASE HELP !
Here is the Schema design i used. I am not sure if this is how this needs to be done so that I can scale it to the data needs that I have. What should I change so that my system does not Hang / Crash ? ( I have a 75 gig HD and 16 GB RAM and 8 CPU server )
PUT event
{
"settings": {
"analysis": {
"normalizer": {
"lc_normalizer": {
"type": "custom",
"char_filter": [],
"filter": ["lowercase", "asciifolding"]
}
}
}
},
"mappings": {
"events": {
"properties": {
"idx": { "type": "keyword","index" : "true" },
"scrip": { "type": "integer", "index" : "true" },
"entered": { "type": "date" , "index" : "true" , "format": "strict_date_optional_time||epoch_millis" },
"enteredDate": { "type": "keyword" , "index" : "true"},
"serverDate": { "type": "keyword" , "index" : "true"},
"customer": { "type": "keyword", "index" : "true" },
"machine": { "type": "keyword", "index" : "true" },
"username": { "type": "text", "index" : "false" },
"clientversion": { "type": "text", "index" : "false" },
"clientsize": { "type": "integer", "index" : "false" },
"priority": { "type": "keyword", "index" : "true" },
"description": { "type": "keyword", "index" : "true" },
"type": { "type": "text", "index" : "false" },
"path": { "type": "text", "index" : "false" },
"executable": { "type": "keyword", "index" : "true" },
"version": { "type": "text", "index" : "false" },
"size": { "type": "integer", "index" : "false" },
"id": { "type": "integer", "index" : "true" },
"windowtitle": { "type": "keyword", "index" : "false" },
"string1": { "type": "keyword", "index" : "true", "normalizer": "lc_normalizer" },
"string2": { "type": "keyword", "index" : "true" },
"text1": { "type": "text", "index" : "true" },
"text2": { "type": "text", "index" : "true" },
"text3": { "type": "text", "index" : "true" },
"text4": { "type": "text", "index" : "true" },
"servertime": { "type": "date" , "index" : "true" , "format": "strict_date_optional_time||epoch_millis" },
"uuid": { "type": "keyword", "index" : "false" },
"Tags": { "type": "keyword", "index" : "true" }
}
}
}
}