Hello!
I created a Spring application and now I'm trying to use Filebeat to send this application's logs to Elasticsearch, so that I can view the information in Kibana.
The problem is that even though I configure the filebeat.yml file, the logs are not being sent to Elasticsearch.
I am using docker-compose to run Elasticsearch, Kibana and Filebeat services.
Does anyone know what the problem could be?
docker-compose.yml
version: '3.8'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:8.12.2
container_name: elasticsearch
ports:
- 9200:9200
- 9300:9300
environment:
discovery.type: "single-node"
bootstrap.memory_lock: true
xpack.security.enabled: false
ES_JAVA_OPTS: "-Xms512m -Xmx512m"
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ../esdata:/usr/share/elasticsearch/data
networks:
- microservices-networkkibana:
image: docker.elastic.co/kibana/kibana:8.12.2
container_name: kibana
restart: always
ports:
- 5601:5601
environment:
ELASTICSEARCH_URL: elasticsearch:9200
ELASTICSEARCH_HOSTS: '["elasticsearch:9200"]'
depends_on:
- elasticsearch
networks:
- microservices-networkfilebeat:
image: docker.elastic.co/beats/filebeat:8.12.2
container_name: filebeat
entrypoint: filebeat -e -strict.perms=false
restart: unless-stopped
volumes:
- ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
depends_on:
- elasticsearch
networks:
- microservices-networknetworks:
microservices-network:
driver: bridge
filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- 'user-service/logs/*.log'
json:
overwrite_keys: true
keys_under_root: true
add_error_key: trueoutput.elasticsearch:
hosts: ["elasticsearch:9200"]
docker logs (Filebeat)
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.215Z","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.(*Beat).configure","file.name":"instance/beat.go","file.line":811},"message":"Home path: [/usr/share/filebeat] Config path: [/usr/share/filebeat] Data path: [/usr/share/filebeat/data] Logs path: [/usr/share/filebeat/logs]","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.585Z","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.(*Beat).configure","file.name":"instance/beat.go","file.line":819},"message":"Beat ID: 2aad5b4b-bca4-466f-ad60-69b8c055ce52","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.589Z","log.logger":"seccomp","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/common/seccomp.loadFilter","file.name":"seccomp/seccomp.go","file.line":125},"message":"Syscall filter successfully installed","service.name":"filebeat","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.589Z","log.logger":"beat","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.logSystemInfo","file.name":"instance/beat.go","file.line":1340},"message":"Beat info","service.name":"filebeat","system_info":{"beat":{"path":{"config":"/usr/share/filebeat","data":"/usr/share/filebeat/data","home":"/usr/share/filebeat","logs":"/usr/share/filebeat/logs"},"type":"filebeat","uuid":"2aad5b4b-bca4-466f-ad60-69b8c055ce52"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.589Z","log.logger":"beat","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.logSystemInfo","file.name":"instance/beat.go","file.line":1349},"message":"Build info","service.name":"filebeat","system_info":{"build":{"commit":"0b71acf2d6b4cb6617bff980ed6caf0477905efa","libbeat":"8.12.2","time":"2024-02-15T13:39:15.000Z","version":"8.12.2"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.589Z","log.logger":"beat","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.logSystemInfo","file.name":"instance/beat.go","file.line":1352},"message":"Go runtime info","service.name":"filebeat","system_info":{"go":{"os":"linux","arch":"amd64","max_procs":4,"version":"go1.20.12"},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.590Z","log.logger":"beat","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.logSystemInfo","file.name":"instance/beat.go","file.line":1358},"message":"Host info","service.name":"filebeat","system_info":{"host":{"architecture":"x86_64","boot_time":"2024-04-26T19:47:27Z","containerized":true,"name":"5dccef9fd287","ip":["127.0.0.1","172.19.0.7"],"kernel_version":"5.15.146.1-microsoft-standard-WSL2","mac":["02:42:ac:13:00:07"],"os":{"type":"linux","family":"debian","platform":"ubuntu","name":"Ubuntu","version":"20.04.6 LTS (Focal Fossa)","major":20,"minor":4,"patch":6,"codename":"focal"},"timezone":"UTC","timezone_offset_sec":0},"ecs.version":"1.6.0"}}
{"log.level":"info","@timestamp":"2024-04-26T20:07:06.591Z","log.logger":"beat","log.origin":{"function":"github.com/elastic/beats/v7/libbeat/cmd/instance.logSystemInfo","file.name":"instance/beat.go","file.line":1387},"message":"Process info","service.name":"filebeat","system_info":{"process":{"capabilities":{"inheritable":null,"permitted":null,"effective":null,"bounding":["chown","dac_override","fowner","fsetid","kill","setgid","setuid","setpcap","net_bind_service","net_raw","sys_chroot","mknod","audit_write","setfcap"],"ambient":null},"cwd":"/usr/share/filebeat","exe":"/usr/share/filebeat/filebeat","name":"filebeat","pid":1,"ppid":0,"seccomp":{"mode":"filter","no_new_privs":true},"start_time":"2024-04-26T20:07:05.520Z"},"ecs.version":"1.6.0"}}
docker logs (Elasticsearch)
CompileCommand: exclude org/apache/lucene/util/MSBRadixSorter.computeCommonPrefixLengthAndBuildHistogram bool exclude = true
CompileCommand: exclude org/apache/lucene/util/RadixSelector.computeCommonPrefixLengthAndBuildHistogram bool exclude = true
Apr 26, 2024 8:07:53 PM sun.util.locale.provider.LocaleProviderAdapter <clinit>
WARNING: COMPAT locale provider will be removed in a future release
{"@timestamp":"2024-04-26T20:07:59.368Z", "log.level": "INFO", "message":"Java vector incubator API enabled; uses preferredBitSize=256; FMA enabled", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.apache.lucene.internal.vectorization.PanamaVectorizationProvider","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:01.648Z", "log.level": "INFO", "message":"version[8.12.2], pid[66], build[docker/48a287ab9497e852de30327444b0809e55d46466/2024-02-19T10:04:32.774273190Z], OS[Linux/5.15.146.1-microsoft-standard-WSL2/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/21.0.2/21.0.2+13-58]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:01.650Z", "log.level": "INFO", "message":"JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:01.651Z", "log.level": "INFO", "message":"JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -Djava.security.manager=allow, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=org.elasticsearch.preallocate, -Des.cgroups.hierarchy.override=/, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-14248558004999123891, --add-modules=jdk.incubator.vector, -XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram, -XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.distribution.type=docker, --module-path=/usr/share/elasticsearch/lib, --add-modules=jdk.net, --add-modules=ALL-MODULE-PATH, -Djdk.module.main=org.elasticsearch.server]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:16.848Z", "log.level": "INFO", "message":"loaded module [repository-url]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:16.849Z", "log.level": "INFO", "message":"loaded module [rest-root]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
{"@timestamp":"2024-04-26T20:08:16.849Z", "log.level": "INFO", "message":"loaded module [x-pack-core]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"84daa2553d9b","elasticsearch.cluster.name":"docker-cluster"}
docker logs (Kibana)
Kibana is currently running with legacy OpenSSL providers enabled! For details and instructions on how to disable see https://www.elastic.co/guide/en/kibana/8.12/production.html#openssl-legacy-provider
{"log.level":"info","@timestamp":"2024-04-25T19:30:43.824Z","log.logger":"elastic-apm-node","ecs.version":"8.10.0","agentVersion":"4.2.0","env":{"pid":8,"proctitle":"/usr/share/kibana/bin/../node/bin/node","os":"linux 5.15.146.1-microsoft-standard-WSL2","arch":"x64","host":"176e3945a26e","timezone":"UTC+00","runtime":"Node.js v18.18.2"},"config":{"active":{"source":"start","value":true},"breakdownMetrics":{"source":"start","value":false},"captureBody":{"source":"start","value":"off","commonName":"capture_body"},"captureHeaders":{"source":"start","value":false},"centralConfig":{"source":"start","value":false},"contextPropagationOnly":{"source":"start","value":true},"environment":{"source":"start","value":"production"},"globalLabels":{"source":"start","value":[["git_rev","f5bd489c5ff9c676c4f861c42da6ea99ae350832"]],"sourceValue":{"git_rev":"f5bd489c5ff9c676c4f861c42da6ea99ae350832"}},"logLevel":{"source":"default","value":"info","commonName":"log_level"},"metricsInterval":{"source":"start","value":120,"sourceValue":"120s"},"serverUrl":{"source":"start","value":"https://kibana-cloud-apm.apm.us-east-1.aws.found.io/","commonName":"server_url"},"transactionSampleRate":{"source":"start","value":0.1,"commonName":"transaction_sample_rate"},"captureSpanStackTraces":{"source":"start","sourceValue":false},"secretToken":{"source":"start","value":"[REDACTED]","commonName":"secret_token"},"serviceName":{"source":"start","value":"kibana","commonName":"service_name"},"serviceVersion":{"source":"start","value":"8.12.2","commonName":"service_version"}},"activationMethod":"require","message":"Elastic APM Node.js Agent v4.2.0"}
[2024-04-25T19:31:26.913+00:00][INFO ][root] Kibana is starting
[2024-04-25T19:31:27.937+00:00][INFO ][node] Kibana process configured with roles: [background_tasks, ui]
[2024-04-25T19:34:23.154+00:00][INFO ][plugins-service] Plugin "cloudChat" is disabled.
[2024-04-25T19:34:23.178+00:00][INFO ][plugins-service] Plugin "cloudExperiments" is disabled.
[2024-04-25T19:34:23.179+00:00][INFO ][plugins-service] Plugin "cloudFullStory" is disabled.
[2024-04-25T19:34:24.297+00:00][INFO ][plugins-service] Plugin "profilingDataAccess" is disabled.
[2024-04-25T19:34:24.297+00:00][INFO ][plugins-service] Plugin "profiling" is disabled.
[2024-04-25T19:34:24.529+00:00][INFO ][plugins-service] Plugin "securitySolutionServerless" is disabled.
[2024-04-25T19:34:24.530+00:00][INFO ][plugins-service] Plugin "serverless" is disabled.
[2024-04-25T19:34:24.531+00:00][INFO ][plugins-service] Plugin "serverlessObservability" is disabled.
[2024-04-25T19:34:24.533+00:00][INFO ][plugins-service] Plugin "serverlessSearch" is disabled.
[2024-04-25T19:34:26.008+00:00][INFO ][http.server.Preboot] http server running at http://0.0.0.0:5601
[2024-04-25T19:34:26.793+00:00][INFO ][plugins-system.preboot] Setting up [1] plugins: [interactiveSetup]
[2024-04-25T19:34:26.945+00:00][INFO ][preboot] "interactiveSetup" plugin is holding setup: Validating Elasticsearch connection configuration…
[2024-04-25T19:34:27.005+00:00][INFO ][root] Holding setup until preboot stage is completed.
[2024-04-25T19:34:27.382+00:00][WARN ][config.deprecation] The default mechanism for Reporting privileges will work differently in future versions, which will affect the behavior of this cluster. Set "xpack.reporting.roles.enabled" to "false" to adopt the future behavior before upgrading.
[2024-04-25T19:34:35.707+00:00][INFO ][plugins-system.standard] Setting up [149] plugins: [devTools,translations,share,screenshotMode,usageCollection,telemetryCollectionManager,telemetryCollectionXpack,taskManager,kibanaUsageCollection,cloud,newsfeed,savedObjectsFinder,noDataPage,monitoringCollection,licensing,mapsEms,globalSearch,globalSearchProviders,features,guidedOnboarding,banners,licenseApiGuard,customBranding,ftrApis,fieldFormats,expressions,screenshotting,esUiShared,customIntegrations,contentManagement,dataViews,home,searchprofiler,painlessLab,management,spaces,security,telemetry,licenseManagement,snapshotRestore,lists,files,encryptedSavedObjects,eventLog,actions,notifications,cloudDataMigration,advancedSettings,grokdebugger,console,bfetch,data,savedObjectsTagging,savedObjectsManagement,unifiedSearch,graph,alerting,embeddable,uiActionsEnhanced,savedSearch,presentationUtil,expressionShape,expressionRevealImage,expressionRepeatImage,expressionMetric,expressionImage,controls,fileUpload,ingestPipelines,ecsDataQualityDashboard,dataViewFieldEditor,dataViewManagement,charts,watcher,visualizations,visTypeXy,visTypeVislib,visTypeVega,visTypeTimeseries,visTypeTimelion,visTypeTagcloud,visTypeTable,visTypeMetric,visTypeMarkdown,visTypeHeatmap,inputControlVis,expressionTagcloud,expressionPartitionVis,visTypePie,expressionMetricVis,expressionLegacyMetricVis,expressionHeatmap,expressionGauge,visTypeGauge,eventAnnotation,expressionXY,dashboard,lens,triggersActionsUi,transform,stackConnectors,stackAlerts,ruleRegistry,cases,timelines,sessionView,kubernetesSecurity,threatIntelligence,metricsDataAccess,aiops,links,discover,reporting,canvas,fleet,osquery,logExplorer,indexManagement,rollup,remoteClusters,crossClusterReplication,indexLifecycleManagement,datasetQuality,cloudSecurityPosture,cloudDefend,discoverEnhanced,maps,dataVisualizer,ml,observabilityAIAssistant,logsShared,observabilityLogExplorer,enterpriseSearch,observability,uptime,synthetics,observabilityOnboarding,infra,upgradeAssistant,monitoring,logstash,elasticAssistant,securitySolution,securitySolutionEss,dashboardEnhanced,apmDataAccess,assetManager,apm,ux]
[2024-04-25T19:34:37.344