Hi,
I am running the standard 7.10.0 Logstash docker container and I want to use it to do its thing and send data on to an ECE elastic deployment.
I'm running into an odd error that I cannot find a solution for. When I start the container it throws the following error [ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
I have logstash.yml and pipelines.yml file configured and mapped to /usr/share/logstash/config, individual cfg files mapped to /usr/share/logstash/pipeline
Here are the files:
## docker-compose.yml
version: "3.7"
services:
logstash:
image: docker.elastic.co/logstash/logstash:7.10.0
container_name: logstash
# restart: unless-stopped
volumes:
- /home/admin/docker/logstash/config/:/usr/share/logstash/config/:ro
- /home/admin/docker/logstash/pipeline/:/usr/share/logstash/pipeline/
- /home/admin/docker/certs/:/usr/share/logstash/config/certs/:ro
ports:
- "60000:8080"
- "5044:5044"
- "9600:9600"
networks:
- elastic-services
metricbeat:
image: docker.elastic.co/beats/metricbeat:7.10.0
container_name: metricbeat
# restart: unless-stopped
command: ["--strict.perms=false"]
user: root
volumes:
- /home/admin/docker/metricbeat/config/metricbeat.docker.yml:/usr/share/metricbeat/metricbeat.yml
- /home/admin/docker/certs:/usr/share/metricbeat/config/certs
- /var/run/docker.sock:/var/run/docker.sock:ro
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
- /proc:/hostfs/proc:ro
- /:/hostfs:ro
network_mode: host
networks:
elastic-services:
name: elastic-services
## logstash.yml
log.level: "debug"
config.debug: true
http.host: 0.0.0.0
node.name: "logstash-1"
# config.reload.automatic: true
monitoring.enabled: false
pipeline.ecs_compatibility: v1
xpack:
management:
enabled: true
elasticsearch:
hosts: https://abcdef0123456789.ece.example.com:9243
username: elastic
password: password
ssl:
certificate_authority: /usr/share/logstash/config/certs/ca_ece.example.com.pem
verification_mode: certificate
## pipelines.yml
- pipeline.id: main
path.config: "/usr/share/logstash/pipeline/test.cfg"
## test.cfg
input {
http {
port => 60000
response_headers => {
"Access-Controle-Allow-Origin" => "*"
"Content-Type" => "application/json"
"Acess-Control-Allow-Headers" => "Origin, X-Requested-With, Content-Type, Accept"
}
}
}
output {
elasticsearch {
hosts => ["https://abcdef0123456789.ece.example.com:9243"]
index => "test"
user => "elastic"
password => "password"
ssl => true
ssl_certificate_verification => none
cacert => "/usr/share/logstash/config/certs/ca_ece.example.com.pem"
}
}
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << HTTP/1.1 200 OK
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << Content-Encoding: gzip
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << Content-Length: 93
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << Content-Type: application/json; charset=UTF-8
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << X-Cloud-Request-Id: H9D2FmzqQYG-BOt_ttc0pA
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << X-Found-Handling-Cluster: 988449793d6040a3aae51e71dcbf2afe
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << X-Found-Handling-Instance: instance-0000000000
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.headers ] http-outgoing-3 << Date: Tue, 08 Dec 2020 16:51:45 GMT
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.impl.execchain.MainClientExec] Connection can be kept alive indefinitely
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.impl.auth.HttpAuthenticator] Authentication succeeded
logstash | [2020-12-08T16:51:45,122][DEBUG][org.apache.http.impl.conn.PoolingHttpClientConnectionManager] Connection [id: 3][route: {s}->https://abcdef0123456789.ece.example.com:9243] can be kept alive indefinitely
logstash | [2020-12-08T16:51:45,123][DEBUG][org.apache.http.impl.conn.PoolingHttpClientConnectionManager] Connection released: [id: 3][route: {s}->https://abcdef0123456789.ece.example.com:9243][total kept alive: 1; route allocated: 1 of 100; total allocated: 1 of 1000]
logstash | [2020-12-08T16:51:45,123][DEBUG][logstash.configmanagement.elasticsearchsource] Could not find a remote configuration for a specific `pipeline_id` {:pipeline_id=>"main"}
logstash | [2020-12-08T16:51:45,123][ERROR][logstash.config.sourceloader] No configuration found in the configured sources.
logstash | [2020-12-08T16:51:45,123][DEBUG][logstash.agent ] Converging pipelines state {:actions_count=>0}
logstash | [2020-12-08T16:51:49,887][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ParNew"}
logstash | [2020-12-08T16:51:49,887][DEBUG][logstash.instrument.periodicpoller.jvm] collector name {:name=>"ConcurrentMarkSweep"}
logstash | [2020-12-08T16:51:50,111][DEBUG][org.apache.http.client.protocol.RequestAuthCache] Re-using cached 'basic' auth scheme for https://abcdef0123456789.ece.example.com:9243
logstash | [2020-12-08T16:51:50,111][DEBUG][org.apache.http.impl.conn.PoolingHttpClientConnectionManager] Connection request: [route: {s}->https://abcdef0123456789.ece.example.com:9243][total kept alive: 1; route allocated: 1 of 100; total allocated: 1 of 1000]
logstash | [2020-12-08T16:51:50,111][DEBUG][org.apache.http.impl.conn.PoolingHttpClientConnectionManager] Connection leased: [id: 3][route: {s}->https://abcdef0123456789.ece.example.com:9243][total kept alive: 0; route allocated: 1 of 100; total allocated: 1 of 1000]
I'd appreciate any insight into this. I've been going at it for a few hours and just seem to be going around in circles.
Thanks.
[edit] I'm not really sure what's going on with the code formatting, sorry it's a bit funky.