Need help how to index with template in LogStash

Elastic Stack Logstash
1

Hi, i'm new to LogStash and i'm trying to index from a csv. I really can't understand what is going on, tried many solutions but LogStash keep stopping the pipeline.

here's my .conf

input {
    file {
        path => "/home/sparko/datasets/mlol/dataset_editori_2016_completo_elastic.csv"
        start_position => "beginning"
    }
}
filter {
    csv 
    {
	separator => ","	
        columns => [
        "IDEDITORE",
        "NOME",
	"gen",
        "feb",
	"mar",
	"apr",
        "mag",
	"giu",
	"lug",
	"ago",
        "set",
	"ott",
	"nov",
	"dic",
        "TITOLI_UNICI",
	"UTENTI_UNICI",
        "PREST_NARRATIVA",
        "PREST_NON_NARRATIVA",
        "TOT_PRESTITI",
	"PREST_PID",
	"ENTI_UNICI",
	"AUTORI_UNICI",
	"PREST_NO",
	"PEST_NE",
	"PREST_CE",
	"PREST_SU",
	"PREST_ISO",
	"SOMMA_UTENTI_Q1",
	"SOMMA_UTENTI_Q2",
	"SOMMA_UTENTI_Q3",
	"SOMMA_UTENTI_Q4",
	"UTENTI_Q1%",
	"UTENTI_Q2%",
	"UTENTI_Q3%",
	"UTENTI_Q4%"
	]
 }
mutate {

convert => {
            "IDEDITORE" => "integer"
            "NOME" => "text"
            "gen" => "half_float"
            "feb"=>"half_float"
            "mar" => "half_float"
            "apr" => "half_float"
            "mag" => "half_float"
            "giu" => "half_float"
            "lug" => "half_float"
            "ago" => "half_float"
            "set" => "half_float"
            "ott" => "half_float"
            "nov" => "half_float"
            "dic" => "half_float"
            "TITOLI_UNICI" => "integer"
            "UTENTI_UNICI" => "integer"
            "PREST_NARRATIVA" => "integer"
            "PREST_NON_NARRATIVA" => "integer"
            "TOT_PRESTITI" => "integer"
            "PREST_PID" => "integer"
            "ENTI_UNICI" => "integer"
            "AUTORI_UNICI" => "integer"
            "PREST_NO" => "half_float"
            "PREST_NE" => "half_float"
            "PREST_CE" => "half_float"
            "PREST_SU" => "half_float"
            "PREST_ISO" => "half_float"
            "SOMMA_UTENTI_Q1" => "half_float"
            "SOMMA_UTENTI_Q2" => "half_float"
            "SOMMA_UTENTI_Q3" => "half_float"
            "SOMMA_UTENTI_Q4" => "half_float"
            "UTENTI_Q1%" => "half_float"
            "UTENTI_Q2%" => "half_float"
            "UTENTI_Q3%" => "half_float"
            "UTENTI_Q4%" => "half_float"
        }

    }
}
output {    
	elasticsearch {
    	hosts => ["localhost:9200"]
    	index => "mloldata"
    	}
    	stdout {}
}

i'm starting LogStash with:

sudo ./bin/logstash -f /home/sparko/datasets/mlol/utils/mlol.conf --path.settings=/etc/logstash/logstash.yml --verbose

and i always get this message:

15:16:08.747 [[main]-pipeline-manager] ERROR logstash.agent - Pipeline aborted due to error {:exception=>#<LogStash::ConfigurationError: translation missing: en.logstash.agent.configuration.invalid_plugin_register>, :backtrace=>["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:178:in `register'", "org/jruby/RubyHash.java:1342:in `each'", "/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-filter-mutate-3.1.3/lib/logstash/filters/mutate.rb:172:in `register'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:268:in `register_plugin'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:279:in `register_plugins'", "org/jruby/RubyArray.java:1613:in `each'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:279:in `register_plugins'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:289:in `start_workers'", "/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:214:in `run'", "/usr/share/logstash/logstash-core/lib/logstash/agent.rb:398:in `start_pipeline'"]}
15:16:08.796 [Api Webserver] INFO  logstash.agent - Successfully started Logstash API endpoint {:port=>9600}
15:16:11.759 [LogStash::Runner] WARN  logstash.agent - stopping pipeline {:id=>"main"}

I've put my template in Kibana console, don't know if i made some mistakes, but i used:

PUT _template/mloldata
{
"template": "mloldata",
  "settings": {
    "number_of_shards": 1
  },
  "mappings": {
    "editore": {
      "properties": {
        "IDEDITORE":{"type": "integer"},
        "NOME":{"type":"half_float"},
        "feb": {"type":"half_float"},
    "mar":{"type":"half_float"},
    "apr":{"type":"half_float"},
    "mag":{"type":"half_float"},
    "giu":{"type":"half_float"},
    "lug":{"type":"half_float"},
    "ago":{"type":"half_float"},
    "set":{"type":"half_float"},
    "ott":{"type":"half_float"},
    "nov":{"type":"half_float"},
    "dic":{"type":"half_float"},
    "TITOLI_UNICI":{"type":"integer"},
    "UTENTI_UNICI":{"type":"integer"},
    "PREST_NARRATIVA":{"type":"integer"},
    "PREST_NON_NARRATIVA":{"type":"integer"},
    "TOT_PRESTITI":{"type":"integer"},
    "PREST_PID":{"type":"integer"},
    "ENTI_UNICI":{"type":"integer"},
    "AUTORI_UNICI":{"type":"integer"},
    "PREST_NO":{"type":"half_float"},
    "PREST_NE":{"type":"half_float"},
    "PREST_CE":{"type":"half_float"},
    "PREST_SU":{"type":"half_float"},
    "PREST_ISO":{"type":"half_float"},
    "SOMMA_UTENTI_Q1":{"type":"half_float"},
    "SOMMA_UTENTI_Q2":{"type":"half_float"},
    "SOMMA_UTENTI_Q3":{"type":"half_float"},
    "SOMMA_UTENTI_Q4":{"type":"half_float"},
    "UTENTI_Q1%":{"type":"half_float"},
    "UTENTI_Q2%":{"type":"half_float"},
    "UTENTI_Q3%":{"type":"half_float"},
    "UTENTI_Q4%":{"type":"half_float"}
      }
    }
  }
}

I'm really lost here, thanks.

2

never mind, problem solved :grin:

3

Solved how?

4

Hi, for now i changed my conf to this:

input {
    file {
        path => "/home/sparko/datasets/mlol/dataset_editori_2016_completo_elastic.csv"
        start_position => "beginning"
	sincedb_path => "/dev/null"
    }
}
filter {
    csv 
    {
	separator => ","	
        columns => [
        "IDEDITORE",
        "NOME",
	"gen",
        "feb",
	"mar",
	"apr",
        "mag",
	"giu",
	"lug",
	"ago",
        "set",
	"ott",
	"nov",
	"dic",
        "TITOLI_UNICI",
	"UTENTI_UNICI",
        "PREST_NARRATIVA",
        "PREST_NON_NARRATIVA",
        "TOT_PRESTITI",
	"PREST_PID",
	"ENTI_UNICI",
	"AUTORI_UNICI",
	"PREST_NO",
	"PEST_NE",
	"PREST_CE",
	"PREST_SU",
	"PREST_ISO",
	"SOMMA_UTENTI_Q1",
	"SOMMA_UTENTI_Q2",
	"SOMMA_UTENTI_Q3",
	"SOMMA_UTENTI_Q4",
	"UTENTI_Q1%",
	"UTENTI_Q2%",
	"UTENTI_Q3%",
	"UTENTI_Q4%"
	]
      }

    }
output {    
	elasticsearch {
    	action => "index"
	hosts => ["localhost:9200"]
	index => "mloldata"
	template => "/home/sparko/datasets/mlol/utils/mloltemplate.json"
	document_type => "editore"
	template_name => "mloldata"
	template_overwrite => "true"
    	}
    	stdout {}
}

removing type conversion and setting up a template file without pre putting a template on the elasticsearch side:

template.json file:

{

"template": "mloldata",
  "settings": {
    "number_of_shards": 1
  },
  "mappings": {
    "editore": {
      "properties": {
        "IDEDITORE":{"type": "keyword"},
        "NOME":{
	  "type":"text",
	  "fields": {
	    "raw":{
		"type":"keyword"		
		}
	   }
	},
        "feb": {"type":"integer"},
    "mar":{"type":"integer"},
    "apr":{"type":"integer"},
    "mag":{"type":"integer"},
    "giu":{"type":"integer"},
    "lug":{"type":"integer"},
    "ago":{"type":"integer"},
    "set":{"type":"integer"},
    "ott":{"type":"integer"},
    "nov":{"type":"integer"},
    "dic":{"type":"integer"},
    "TITOLI_UNICI":{"type":"integer"},
    "UTENTI_UNICI":{"type":"integer"},
    "PREST_NARRATIVA":{"type":"integer"},
    "PREST_NON_NARRATIVA":{"type":"integer"},
    "TOT_PRESTITI":{"type":"integer"},
    "PREST_PID":{"type":"integer"},
    "ENTI_UNICI":{"type":"integer"},
    "AUTORI_UNICI":{"type":"integer"},
    "PREST_NO":{"type":"integer"},
    "PREST_NE":{"type":"integer"},
    "PREST_CE":{"type":"integer"},
    "PREST_SU":{"type":"integer"},
    "PREST_ISO":{"type":"integer"},
    "SOMMA_UTENTI_Q1":{"type":"integer"},
    "SOMMA_UTENTI_Q2":{"type":"integer"},
    "SOMMA_UTENTI_Q3":{"type":"integer"},
    "SOMMA_UTENTI_Q4":{"type":"integer"},
    "UTENTI_Q1%":{"type":"double"},
    "UTENTI_Q2%":{"type":"double"},
    "UTENTI_Q3%":{"type":"double"},
    "UTENTI_Q4%":{"type":"double"}
      }
    }
  }
}

Then i relaunched logstash with a slightly different "path.settings" as found here on a github issue (i'm using debian packages on ubuntu server): https://github.com/elastic/logstash/issues/6582

Everything went fine and my index was populated correctly with my custom mapping
Only i had to manually stop logstash because it kept running, but it's almost due to my newbie skill with logstash i think, hope i'll get it right sooner or later.

5

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.