Hello,
I'm trying to create indexes in elasticsearch from a postgresql database.
So I set up docker compose:
version: '3.8'
services:
postgres:
image: postgres:latest
volumes:
- C:\Users\theor\desktop\travail\noota\noot-search\initdb:/docker-entrypoint-initdb.d/
- postgres_data:/var/lib/postgresql/data
environment:
- POSTGRES_PASSWORD=mysecretpassword
ports:
- "5432:5432"
networks:
- elastic-infrastructure-network
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.11.2
environment:
- discovery.type=single-node
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=tidoz!@#6BtAqY7sQSck
- logger.level=ERROR
ports:
- "9200:9200"
- "9300:9300"
networks:
- elastic-infrastructure-network
kibana:
image: docker.elastic.co/kibana/kibana:8.11.2
ports:
- "5601:5601"
depends_on:
- elasticsearch
networks:
- elastic-infrastructure-network
logstash:
image: docker.elastic.co/logstash/logstash:8.11.2
environment:
- LOG_LEVEL=error
volumes:
- ./logstash_configs/logstash-dossiers.conf:/usr/share/logstash/pipeline/logstash-dossiers.conf
- ./logstash_configs/logstash-familytags.conf:/usr/share/logstash/pipeline/logstash-familytags.conf
- C:\Users\theor\desktop\travail\noota\noot-search\jdbc_driver:/usr/share/logstash/jdbc_driver
depends_on:
- postgres
- elasticsearch
networks:
- elastic-infrastructure-network
networks:
elastic-infrastructure-network:
volumes:
postgres_data:
I also installed the jdbc plugin and created configuration files for my indexes:
logstash_configs/logstash-dossiers.conf :
input {
jdbc {
jdbc_driver_library => "/usr/share/logstash/jdbc_driver/postgresql.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://postgres:5432/postgres"
jdbc_user => "postgres"
jdbc_password => "mysecretpassword"
schedule => "0 * * * *"
statement => "SELECT * FROM dossiers"
}
tcp {
port => 50000
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
user => "elastic"
password => "tidoz!@#6BtAqY7sQSck"
index => "dossiers_index"
document_type => "_doc"
}
}
logstash_configs/logstash-familytags.conf :
input {
jdbc {
jdbc_driver_library => "/usr/share/logstash/jdbc_driver/postgresql.jar"
jdbc_driver_class => "org.postgresql.Driver"
jdbc_connection_string => "jdbc:postgresql://postgres:5432/postgres"
jdbc_user => "postgres"
jdbc_password => "mysecretpassword"
schedule => "0 * * * *"
statement => "SELECT * FROM familytags"
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
user => "elastic"
password => "tidoz!@#6BtAqY7sQSck"
index => "familytags"
document_type => "_doc"
}
}
But when I run docker compose, I have the following logs:
Elasticsearch :
2023-12-15 14:02:44 Dec 15, 2023 1:02:44 PM sun.util.locale.provider.LocaleProviderAdapter <clinit>
2023-12-15 14:02:44 WARNING: COMPAT locale provider will be removed in a future release
2023-12-15 14:02:27 Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
Logstash :
2023-12-15 14:02:27 Using bundled JDK: /usr/share/logstash/jdk
2023-12-15 14:02:42 Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
2023-12-15 14:02:45 [2023-12-15T13:02:45,142][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError}
2023-12-15 14:02:45 [2023-12-15T13:02:45,150][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
2023-12-15 14:02:45 [2023-12-15T13:02:45,169][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster.
2023-12-15 14:03:15 [2023-12-15T13:03:15,163][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError}
2023-12-15 14:03:15 [2023-12-15T13:03:15,164][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
2023-12-15 14:03:45 [2023-12-15T13:03:45,162][ERROR][logstash.licensechecker.licensereader] Unable to retrieve Elasticsearch cluster info. {:message=>"No Available connections", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError}
2023-12-15 14:03:45 [2023-12-15T13:03:45,163][ERROR][logstash.licensechecker.licensereader] Unable to retrieve license information from license server {:message=>"No Available connections"}
2023-12-15 14:02:27 2023/12/15 13:02:27 Setting 'log.level' from environment.
I've tried disabling SSL and re-enabled it but nothing works.
I'm a little lost, I don't really know how to solve this problem, if you have a solution I'm interested.
Thank you in advance for your help.
Theo.