I have azure storage container blob in azure with .json file uploaded in it i want automatically json file should be fetched from the blob storage and pushed to elasticsearch using helm how can we acheive it using filebeat values.yaml . I have tried to acheive this below is the values.yaml , but still blob data is not getting downloaded and pushed to elasticsearch. when we run a kubectl get pods , pods shows running state of filebeat .
daemonset:
annotations: {}
labels: {}
affinity: {}
enabled: true
envFrom: []
extraEnvs:
- name: "ELASTICSEARCH_USERNAME"
valueFrom:
secretKeyRef:
name: elasticsearch-master-credentials
key: username
- name: "ELASTICSEARCH_PASSWORD"
valueFrom:
secretKeyRef:
name: elasticsearch-master-credentials
key: password
- name: "AZURE_STORAGE_ACCOUNT"
value: "elktestsa"
- name: "AZURE_CONTAINER_NAME"
value: "elkcontainer"
- name: "AZURE_FILE_NAME"
value: "PT1H 1.json"
- name: "AZURE_STORAGE_KEY"
valueFrom:
secretKeyRef:
name: azure-storage-credentials
key: storage-key
extraVolumes:
- name: filebeat-logs
emptyDir: {}
extraVolumeMounts:
- name: filebeat-logs
mountPath: /usr/share/filebeat/logs
hostNetworking: false
filebeatConfig:
filebeat.yml: |
filebeat.inputs:
- type: log
paths:
- "/usr/share/filebeat/logs/PT1H 1.json"
json.keys_under_root: true
json.add_error_key: true
output.elasticsearch:
hosts: ["https://elasticsearch-master:9200"]
username: "${ELASTICSEARCH_USERNAME}"
password: "${ELASTICSEARCH_PASSWORD}"
protocol: https
ssl.verification_mode: "none"
maxUnavailable: 1
nodeSelector: {}
secretMounts:
- name: elasticsearch-master-certs
secretName: elasticsearch-master-certs
path: /usr/share/filebeat/certs/
securityContext:
runAsUser: 0
privileged: false
resources:
requests:
cpu: "100m"
memory: "100Mi"
limits:
cpu: "1000m"
memory: "200Mi"
tolerations: []
extraInitContainers:
- name: fetch-json-from-blob
image: mcr.microsoft.com/azure-cli
command:
- /bin/sh
- -c
- |
echo "Starting batch download of JSON files from Azure Blob Storage..."
if az storage blob download-batch \
--source "elkcontainer" \
--destination "./" \
--account-name "elktestsa" \
--account-key "ziHOLDdQFQPpN+jjOxwyhvXGDBasbz5rim6UJ9Yzo/ycPwz0c1JgZi3lIpDTz8vbzvgVvdxjHsYI+ASt8uJPnA=="; then
echo "Batch download completed successfully."
else
echo "Batch download failed. Check Azure credentials, container name, or file path."
fi
volumeMounts:
- mountPath: /usr/share/filebeat/logs
name: filebeat-logs
deployment:
annotations: {}
labels: {}
affinity: {}
enabled: false
envFrom: []
extraEnvs:
- name: "ELASTICSEARCH_USERNAME"
valueFrom:
secretKeyRef:
name: elasticsearch-master-credentials
key: username
- name: "ELASTICSEARCH_PASSWORD"
valueFrom:
secretKeyRef:
name: elasticsearch-master-credentials
key: password
filebeatConfig:
filebeat.yml: |
filebeat.inputs:
- type: log
paths:
- /usr/share/filebeat/logs/filebeat
output.elasticsearch:
hosts: ["https://${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}"]
username: "${ELASTICSEARCH_USERNAME}"
password: "${ELASTICSEARCH_PASSWORD}"
protocol: https
nodeSelector: {}
secretMounts:
- name: elasticsearch-master-certs
secretName: elasticsearch-master-certs
path: /usr/share/filebeat/certs/
securityContext:
runAsUser: 0
privileged: false
resources:
requests:
cpu: "100m"
memory: "100Mi"
limits:
cpu: "1000m"
memory: "200Mi"
tolerations: []
replicas: 1
extraContainers: ""
extraInitContainers: []
hostPathRoot: /var/lib
dnsConfig: {}
hostAliases: []
image: "docker.elastic.co/beats/filebeat"
imageTag: "8.5.1"
imagePullPolicy: "IfNotPresent"
imagePullSecrets: []
livenessProbe:
exec:
command:
- sh
- -c
- curl --fail 127.0.0.1:5066
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
exec:
command:
- sh
- -c
- filebeat test output
failureThreshold: 3
initialDelaySeconds: 10
periodSeconds: 10
timeoutSeconds: 5
managedServiceAccount: true
clusterRoleRules:
- apiGroups:
- ""
resources:
- namespaces
- nodes
- pods
verbs:
- get
- list
- watch
- apiGroups:
- "apps"
resources:
- replicasets
verbs:
- get
- list
- watch
podAnnotations: {}
serviceAccount: ""
serviceAccountAnnotations: {}
terminationGracePeriod: 30
priorityClassName: ""
updateStrategy: RollingUpdate
nameOverride: ""
fullnameOverride: ""