Issue in loading CSV file to Elastic search because of \n using LS 7

Elastic Stack Logstash
1

I am loading csv file using LS 7 to Elastic cloud. the document contain we message column that contains \n character within the message content.
below is the configuration file

input {
file {
path => "D:/AzureTables/STG/MT*.csv"
start_position => "beginning"
sincedb_path => "NULL"}}
filter {
mutate {gsub => ["message" , "[\n]"," "]}
csv {separator => "|"
quote_char => "^"
columns => ['PartitionKey','mTMessageLogId','Timestamp','dateCreated','essage','msisdn','operatorCode','merchantId','sMSId','sMSType','sent','shortCode','TransactionId','subscriptionContractId','DateCreatedDMY'] }
date {
match => ["Timestamp", "dd/MM/yyyy HH:mm:ss.SSS"]
    target => "@timestamp"}
date {match => ["dateCreated", "dd/MM/yyyy HH:mm:ss.SSS"]
            target => "dateCreated"    }
mutate { remove_field => ["PartitionKey","TransactionId","Timestamp","host","path","@version","@timestamp"] }
mutate {convert => ["merchantId", "integer"]}
mutate {convert => ["sent", "boolean"]}
mutate {convert => ["subscriptionContractId", "integer"]}}
output {	elasticsearch {		hosts => 
		index => "mtmessagelogs-%{DateCreatedDMY}"
		document_id =>
		user => 
		password =>	}	stdout {}}

below is the issue I am getting

[ERROR][logstash.outputs.elasticsearch][main] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>"%{mTMessageLogId}", :_index=>"mtmessagelogs-%{DateCreatedDMY}", :routing=>nil, :_type=>"_doc"}, #<LogStash::Event:0x777151c8>], :response=>{"index"=>{"_index"=>"mtmessagelogs-%{DateCreatedDMY}", "_type"=>"_doc", "_id"=>"%{mTMessageLogId}", "status"=>400, "error"=>{"type"=>"invalid_index_name_exception", "reason"=>"Invalid index name [mtmessagelogs-%{DateCreatedDMY}], must be lowercase", "index_uuid"=>"_na_", "index"=>"mtmessagelogs-%{DateCreatedDMY}"}}}}
D:/Raef/Logstash/logstash-7.6.1/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
    "message" => "??? ??? ????? ????? ?? : 3.15 ?.?/??? (???? ????? ?????? ???????)."
}
{
    "message" => "?????? ????? ???? ???? ???? ????? ??????? ?? ???? ????? "
}
{
       "dateCreated" => 2019-09-08T08:11:19.393Z,
            "essage" => "???? ?? ???? ???? ?? 80,000 ?.? ? ??????? ???????? 100,000 ???? ?????",
    "mTMessageLogId" => "971561929154-1238304699",
           "message" => "2019-10265|971561929154-1238304699|08/09/2019 10:11:19.528|08/09/2019 10:11:19.393|???? ?? ???? ???? ?? 80,000 ?.? ? ??????? ???????? 100,000 ???? ?????"
}
{
             "sMSId" => "118865134",
            "msisdn" => "SubscriptionCaringMessage",
              "tags" => [
        [0] "_dateparsefailure"
    ],
        "merchantId" => 1113,
           "message" => "???? ???? ????? ??????!|971561929154|42402|10265|1238304699|SubscriptionCaringMessage|true|1113|118865134||20190908",
              "sent" => "20190908",
       "dateCreated" => "10265",
            "essage" => "1238304699",
    "mTMessageLogId" => "971561929154",
      "operatorCode" => "true",
           "sMSType" => nil
}
{
    "message" => "?????? ??????? ????b WinCash??? 7726"
}
{
    "message" => "???????? ???? ??? ??? 1113"
}
{
                     "sMSId" => "sMSId",
                    "msisdn" => "Msisdn",
                      "tags" => [
        [0] "_dateparsefailure"
    ],
                "merchantId" => 0,
                   "message" => "PartitionKey|RowKey|Timestamp|DateCreated|Message|Msisdn|OperatorCode|merchantId|sMSId|SMSType|Sent|ShortCode|TransactionId|subscriptionContractId|DateCreatedDMY",
                      "sent" => "Sent",
               "dateCreated" => "DateCreated",
                    "essage" => "Message",
            "mTMessageLogId" => "RowKey",
    "subscriptionContractId" => 0,
            "DateCreatedDMY" => "DateCreatedDMY",
              "operatorCode" => "OperatorCode",
                 "shortCode" => "ShortCode",
                   "sMSType" => "SMSType"
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.