How to send a specific index to a different ES cluster?

[hadi]

Hello,

I am planning to send Kafka logs to a separate ES cluster, kafka.xxx.com:9200, by the following configuration but unfortunately, it doesn't work.

input {
  kafka {
    bootstrap_servers => "aaa:9092"
    topics => ["sales"]
    add_field => {"type" => "kafka"}
    add_field => {"inputSource" => "kafka"}
    add_field => {"inputType" => "sales"}
    add_field => {"myIndex" => "kafka-sales"}
  }
}

filter {
  if [type] == "kafka" {
  ...
}
output {
  if [type] == "kafka" {
    elasticsearch {
      hosts => ["kafka.xxx.com:9200"]
      manage_template => false
      index => "logstash-playground-%{myIndex}-%{+YYYY.MM.dd}"
    }
  } else {
    elasticsearch {
      hosts => ["log.yyy.com:9200"]
      manage_template => false
      index => "logstash-playground-%{myIndex}-%{+YYYY.ww}"
    }
  }
}

Please let me know if you see my mistake in the output section of the above-mentioned configuration?

[Christian_Dahlqvist]

Where is type set to kafka on the events?

[hadi]

Sorry I didn't get your point, can you please elaborate it a bit?

[hadi]

Can someone help me on this issue please?

[Jasonespo (JasonE)]

Try this:

input {
      kafka {
        bootstrap_servers => "aaa:9092"
        topics => ["sales"]
      }
    }

filter {
      if [message] =~ /Message that Kafka Logs will match against/ {
    
mutate {
 
    add_field => {"type" => "kafka"}
    add_field => {"inputSource" => "kafka"}
    add_field => {"inputType" => "sales"}
    add_field => {"myIndex" => "kafka-sales"}

    }
}

output {
      if [type] == "kafka" {
        elasticsearch {
          hosts => ["kafka.xxx.com:9200"]
          manage_template => false
          index => "logstash-playground-%{myIndex}-%{+YYYY.MM.dd}"
        }
      } else {
        elasticsearch {
          hosts => ["log.yyy.com:9200"]
          manage_template => false
          index => "logstash-playground-%{myIndex}-%{+YYYY.ww}"
        }
      }
    }