Logstash creating an array when I add a string field that not exist yet

Elastic Stack Logstash
1

Hello everyone
I have a logstash conf file that fetch two different queries from different tables in mysql and both are merged in the same index of elasticsearch. One specific string field (code_serial) exists in one table and not in another, so I made a condition that check if that field exists, and if not, I add it with mutate filter and set a message to it. The problem is, when I am setting this message, the new field is converted to array. Why is this happing? I just want a simple string field.

Sample of my conf file below and the results when the field exists and when it's not.

Conf:

input {
  jdbc {
    jdbc_driver_library => "/path/to/driver/mysql-connector-java.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://my-host/db"
    jdbc_user => "user"
    jdbc_password => "pass"
    last_run_metadata_path => "/path/last_run/table1_logstash_jdbc_last_run"
    statement => "SELECT *  FROM table1 WHERE updated_at > CONVERT_TZ(:sql_last_value,'+00:00','-2:00')"
    tags => "table1"
  }

  jdbc {
    jdbc_driver_library => "/path/to/driver/mysql-connector-java.jar"
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://my-host/db"
    jdbc_user => "user"
    jdbc_password => "pass"
    last_run_metadata_path => "/path/last_run/table2_logstash_jdbc_last_run"
    statement => "SELECT * FROM table2 WHERE updated_at > CONVERT_TZ(:sql_last_value,'+00:00','-2:00')"
    tags => "table2"
  }
}

filter {
    if ![code_serial] {
        mutate { add_field => { "code_serial" => "Não Informado" } }
    } else if  [code_serial] == "" {
        mutate { update => { "code_serial" => "Não Informado" } }
    }
}

output {
    if "table1" in [tags] {
    	elasticsearch {
            hosts => "localhost:9200"
    		template => "/path/to/template/template.json"
    		template_overwrite => true
    		index => "my_index"
    		document_id => "%{id1}"
               document_type => "my_doc"
    	}
    }

    if "table2" in [tags] {
        elasticsearch { 
             hosts => "localhost:9200"
             index => "my_index"
             document_id => "%{id2}"
             document_type => "my_doc"
        }
    }
}

Ps: There's no mapping for this field in template file.
Ps2: After running, the field is dinamically mapped to string.

  "code_serial": {
    "type": "text",
    "norms": false,
    "fields": {
      "keyword": {
        "type": "keyword"
      }
    }
  }
  • Result when the field exist in the table
      "code_serial": "12345678910"
  • Result when the field NOT exist in the table and is manually created
      "code_serial": [
        "Não Informado"
      ]
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.