Logs from AWS S3 does not go to AWS CoudWatch

[makimaki (Nikola Malezanoski)]

Hi,

I'm trying to read logs from the S3 bucket and put them to CloudWatch. But nothing happens in the CloudWatch.
Log file in S3 contain min 2 JSON strings

{"docker":{"container_id":"fe98027af6490c542e8d0321115f4c1e9ea3c73e75efee847972372b3273706b"},"kubernetes":{"container_name":"pairwise-individual","namespace_name":"sso-dev","pod_name":"pairwise-individual-44-gtv4q","container_image":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","container_image_id":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","pod_id":"b7922903-809a-462e-bc76-b0704b036727","pod_ip":"10.128.2.11","host":"ip-10-100-142-119.ap-southeast-2.compute.internal","labels":{"application":"pairwise-service","deployment":"pairwise-individual-44","deploymentConfig":"pairwise-individual","deploymentconfig":"pairwise-individual"},"master_url":"https://kubernetes.default.svc","namespace_id":"25ee911c-2b83-424f-88e4-e5f85f7535c1","namespace_labels":{"external":"true","app_kubernetes_io/instance":"sso-dev","kubernetes_io/metadata_name":"sso-dev"}},"message":"2022-01-05 12:43:39.332  INFO [,,,] 1 --- [           main] trationDelegate$BeanPostProcessorChecker : Bean 'org.springframework.cloud.autoconfigure.ConfigurationPropertiesRebinderAutoConfiguration' of type [org.springframework.cloud.autoconfigure.ConfigurationPropertiesRebinderAutoConfiguration] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying)","level":"info","hostname":"ip-10-100-142-119.ap-southeast-2.compute.internal","pipeline_metadata":{"collector":{"ipaddr4":"10.100.142.119","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2022-01-05T12:43:39.796685+00:00","version":"1.7.4 1.6.0"}},"@timestamp":"2022-01-05T12:43:39.335718+00:00","viaq_index_name":"app-write","viaq_msg_id":"ZmIzMDRiYTEtYTQwOS00NTk1LWE5YmYtZWNiMDdlMDkxNzM0","log_type":"application"}
{"docker":{"container_id":"fe98027af6490c542e8d0321115f4c1e9ea3c73e75efee847972372b3273706b"},"kubernetes":{"container_name":"pairwise-individual","namespace_name":"sso-dev","pod_name":"pairwise-individual-44-gtv4q","container_image":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","container_image_id":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","pod_id":"b7922903-809a-462e-bc76-b0704b036727","pod_ip":"10.128.2.11","host":"ip-10-100-142-119.ap-southeast-2.compute.internal","labels":{"application":"pairwise-service","deployment":"pairwise-individual-44","deploymentConfig":"pairwise-individual","deploymentconfig":"pairwise-individual"},"master_url":"https://kubernetes.default.svc","namespace_id":"25ee911c-2b83-424f-88e4-e5f85f7535c1","namespace_labels":{"external":"true","app_kubernetes_io/instance":"sso-dev","kubernetes_io/metadata_name":"sso-dev"}},"message":"","level":"unknown","hostname":"ip-10-100-142-119.ap-southeast-2.compute.internal","pipeline_metadata":{"collector":{"ipaddr4":"10.100.142.119","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2022-01-05T12:43:39.796816+00:00","version":"1.7.4 1.6.0"}},"@timestamp":"2022-01-05T12:43:39.499810+00:00","viaq_index_name":"app-write","viaq_msg_id":"ZmY4MDFlYTAtOGRkNi00NmM2LWI5MmItY2UwZGI5OTczYTNk","log_type":"application"}
{"docker":{"container_id":"fe98027af6490c542e8d0321115f4c1e9ea3c73e75efee847972372b3273706b"},"kubernetes":{"container_name":"pairwise-individual","namespace_name":"sso-dev","pod_name":"pairwise-individual-44-gtv4q","container_image":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","container_image_id":"image-registry.openshift-image-registry.svc:5000/sso-dev/pairwise-service@sha256:1db7f80eef580223d24ada09347a6548e42ccd2c2771fbc0538c8110306c3902","pod_id":"b7922903-809a-462e-bc76-b0704b036727","pod_ip":"10.128.2.11","host":"ip-10-100-142-119.ap-southeast-2.compute.internal","labels":{"application":"pairwise-service","deployment":"pairwise-individual-44","deploymentConfig":"pairwise-individual","deploymentconfig":"pairwise-individual"},"master_url":"https://kubernetes.default.svc","namespace_id":"25ee911c-2b83-424f-88e4-e5f85f7535c1","namespace_labels":{"external":"true","app_kubernetes_io/instance":"sso-dev","kubernetes_io/metadata_name":"sso-dev"}},"message":"  .   ____          _            __ _ _","level":"unknown","hostname":"ip-10-100-142-119.ap-southeast-2.compute.internal","pipeline_metadata":{"collector":{"ipaddr4":"10.100.142.119","inputname":"fluent-plugin-systemd","name":"fluentd","received_at":"2022-01-05T12:43:39.796887+00:00","version":"1.7.4 1.6.0"}},"@timestamp":"2022-01-05T12:43:39.499810+00:00","viaq_index_name":"app-write","viaq_msg_id":"NTEwYjM4MDAtNDlmZi00N2U3LWI3YWItYjdkOWNmMjdiMzVl","log_type":"application"}

This is my config

input {
  s3 {
    id => "idexnpe"
    access_key_id => "xxxx"
    secret_access_key => "xxxxx"
    bucket => "bucket name"
    prefix => "logs"
    region => "ap-southeast-2"
    interval => "10"
    codec => "json_lines"
  }
}

filter {
  alter {
    add_field => { "CW_metric" => "IDXLogs" }
  }
}
output {
  stdout {
    codec => json
  }
  cloudwatch {
    access_key_id => "asdasd"
    secret_access_key => "asdasdasd"
    region => "ap-southeast-2"
  }
  # for debug
  stdout { codec => json }
}

I can see records in STDOUT but not in CloudWatch

Can someone please give me advice