Input JSON file and change type of fields

Elastic Stack Logstash
1

Hi,
I have a json file with a fixed structure and dynamic structure. In the fixed struture I changed the type of data using a template, this part is easy.

Logstash.conf

input {
  stdin {
    codec => json
    }
}
filter {
  date {
      match => [ "FECHA-OPRCN-F", "ISO8601", "YYYY-MM-dd HH:mm:ss" ]
      target => "FECHA-OPRCN-F"
      locale => "en"
    }

}

output {

stdout  {

codec => dots {}

}

elasticsearch {

index => "nginx_json_elastic_stack_example"

document_type => "logs"

template => "./nginx_json_template.json"

template_name => "nginx_json_elastic_stack_example"

template_overwrite => true

}

}

Template

{
  "template": "nginx_json_elastic_stack_example",
  "settings": {
     "index.refresh_interval": "5s"
  },
  "mappings": {
     "_default_": {        
        "properties": {
           "CANAL" : {
				"type" : "long"
			},
			"NUM-PARTICION-F" : {
				"type" : "long"
			},
			"COD-NRBE-EN-F" : {
				"type" : "long"
			},
			"COD-NRBE-EN-FSC-F" : {
				"type" : "long"
			},
			"COD-INTERNO-UO-F" : {
				"type" : "long"
			},
			"COD-INTERNO-UO-FSC-F" : {
				"type" : "long"
			},
			"COD-CSB-OF-F" : {
				"type" : "long"
			},
			"ID-INTERNO-EMPL-EP-F" : {
				"type" : "long"
			},
			"NUM-SEC-F" : {
				"type" : "long"
			},
			"FECHA-CTBLE-F" : {
				"type" : "date"
			},
			"VALOR-IMP-F" : {
				"type" : "double"
			},
			"VARIABLE": {
				"properties": {				  
				  "FECHA-CTBLE-F": {
					"type": "date"
				  }
				}
		    }
        },
        "_all": {
           "enabled": true
        }
     }
  }
}

But I have a problem in the dynamic structure, cause I need to change the type of data inside of "VARIABLE". This structure has many variables, I tried dynamic_templates like a API sample but it didnt work. I wonder if somone could help me, I need to change variables with numbers to numeric type (String->Numeric) in the dynamic structure.

static structure: all variables: NUM-PARTICION-F, NOMBRE-REGLA-F....VALOR-IMP-F, CANAL.
dynamic structure: VARIABLE {... all variables... }

Here I put three lines of me file:

{"NUM-PARTICION-F":"001","NOMBRE-REGLA-F":"TR_POSICION_GLOBAL_RVA_TRN","FECHA-OPRCN-F":"2016-10-16 19:46:30","COD-NRBE-EN-F":"0000","COD-NRBE-EN-FSC-F":"0000","COD-INTERNO-UO-F":"9601","COD-INTERNO-UO-FSC-F":"9601","COD-CSB-OF-F":"0107","COD-CENT-UO-F":"","ID-INTERNO-TERM-TN-F":"A97I9999","ID-INTERNO-EMPL-EP-F":"99999998","NUM-SEC-F":"0","COD-TX-F":"BDPPM1KK","COD-TX-DI-F":"TUX","ID-EMPL-AUT-F":"","FECHA-CTBLE-F":"2016-10-17","HORA-OPRCN-F":"19.46.30","COD-IDENTIFICACION-F":"","IDENTIFICACION-F":"","VALOR-IMP-F":"0.00","CANAL":"03","VARIABLE":{"COD-NRBE-EN-F":"0000","ID-INTERNO-PE-F":"562971","CCC-1-F (0)":"00000000000000","CCC-1-F (1)":"0000000000000000","CCC-1-F (2)":"00000000000","CCC-1-F (3)":"0000000000","CCC-1-F (4)":"0000000000000000000","CCC-1-F (5)":"000000026065904226","CCC-1-F (6)":"000000000000000","CCC-1-F (7)":"00000000000","CCC-1-F (8)":"000000000000000000000","CCC-1-F (9)":"000000000","CCC-1-F (10)":"00000000000","CCC-1-F (11)":"000000000000","CCC-1-F (12)":"000000000","CCC-1-F (13)":"0000000000000","CCC-1-F (14)":"0000000000000","CCC-1-F (15)":"000000000000","CCC-1-F (16)":"000000000000000000"}}

{"NUM-PARTICION-F":"001","NOMBRE-REGLA-F":"TR_ACTU_LIBTA_MPAGO_TRN","FECHA-OPRCN-F":"2016-10-16 19:46:31","COD-NRBE-EN-F":"0000","COD-NRBE-EN-FSC-F":"9998","COD-INTERNO-UO-F":"0001","COD-INTERNO-UO-FSC-F":"0001","COD-CSB-OF-F":"0001","COD-CENT-UO-F":"","ID-INTERNO-TERM-TN-F":"A0299997","ID-INTERNO-EMPL-EP-F":"99999998","NUM-SEC-F":"0","COD-TX-F":"DVI97MOU","COD-TX-DI-F":"TUX","ID-EMPL-AUT-F":"","FECHA-CTBLE-F":"2016-10-17","HORA-OPRCN-F":"19.46.31","COD-IDENTIFICACION-F":"","IDENTIFICACION-F":"","VALOR-IMP-F":"0.00","CANAL":"06","VARIABLE":{"COD-NRBE-EN-F":"0000","COD-CENT-UO-F":"0302","NUM-SEC-AC-F":"0000000","SGN-CTBLE-F":"+","COD-NUMRCO-MONEDA-F":"978","NUM-LINEAS-LB-F":"21","NUM-PAGINAS-LB-F":"2"}}

{"NUM-PARTICION-F":"001","NOMBRE-REGLA-F":"TR_IMPUTAC_MPAGO_TRN","FECHA-OPRCN-F":"2016-10-16 19:46:32","COD-NRBE-EN-F":"0000","COD-NRBE-EN-FSC-F":"0000","COD-INTERNO-UO-F":"0001","COD-INTERNO-UO-FSC-F":"0001","COD-CSB-OF-F":"0001","COD-CENT-UO-F":"","ID-INTERNO-TERM-TN-F":"A0299997","ID-INTERNO-EMPL-EP-F":"99999998","NUM-SEC-F":"0","COD-TX-F":"DVI82OOU","COD-TX-DI-F":"TUX","ID-EMPL-AUT-F":"","FECHA-CTBLE-F":"2016-10-17","HORA-OPRCN-F":"19.46.32","COD-IDENTIFICACION-F":"","IDENTIFICACION-F":"","VALOR-IMP-F":"0.00","CANAL":"06","VARIABLE":{"COD-NRBE-EN-F":"0000","NUM-SEC-AC-F":"000000000","TIPO-OPRCN-F":"C","STD-CHAR-08-F":"16102016","IMP-NOMINAL-F":"11.18","COD-CLOP-SIST-F":"47","TIPO-SBCLOP-F":"0011","TEXTO-REMITENTE-F":"some text example","COD-NUMRCO-MONEDA-F":"978","TIPO-IMPUTAC-F":"0","COD-INTERNO-UO-F":"9330"}}
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.