Beautifying the awslamda nested logs Using Logstash in Kibana

Suman_ISMT · April 26, 2024, 5:30am

Hello Community,

I am using the following Logstash configuration to monitor the aws serverless logs using elasticsearch and visualize it in kibana. The configuration is working no issue on that.

input {
  cloudwatch_logs {
    log_group => ["/aws/lambda/RippleCheckReturnedTransactionLambda"]
    region => "ap-south-1"
    access_key_id => "**************************"
    secret_access_key => "**********************************"
  }
}

filter {
  json {
    source => "message"
    remove_field => ["message"]
  }

  # Flatten the nested JSON structure
  ruby {
    code => '
      content = event.get("[sort][content]")
      contract = content["contract"]
      event.set("[sort][content]", content.to_json)
      event.set("[contract]", contract.to_json)
    '
  }
}

output {
  elasticsearch {
    index => "awscloudwatch"
    ssl => true
    cacert => "/home/******/ca_logstash.cer"
    ssl_certificate_verification => true
    hosts => ["https://***.***.**.**:9200"]
    user => "elastic"
    password => "********"
  }
}

With this configuration I am trying to beautify the message field. But still I am not able to beautify the json formated message field.

Image:

From the above picture I want the message field inside of message string field to be displayed as follows:

{
  "first": true,
  "last": true,
  "number": "0",
  "numberOfElements": "1",
  "size": "9",
  "totalElements": "1",
  "totalPages": "1",
  "sort": [
    {
      "direction": "DESC",
      "property": "MODIFIED_AT",
      "ignoreCase": false,
      "nullHandling": "NATIVE",
      "ascending": false,
      "descending": true
    }
  ],
  "content": [
    {
      "payment_id": "b1e6e6c9-3327-4397-82f0-320cb1a432e1",
      "contract_hash": "2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807",
      "payment_state": "COMPLETED",
      "modified_at": "2024-03-28T05:57:10.540Z",
      "contract": {
        "sender_end_to_end_id": "IpayPinc10011",
        "created_at": "2024-03-28T05:54:22.671Z",
        "expires_at": "2024-03-28T06:46:23.907Z",
        "quote": {
          "quote_id": "3ae96871-72c9-4128-a987-6d2432a8314f",
          "created_at": "2024-03-28T05:46:23.907Z",
          "expires_at": "2024-03-28T06:46:23.907Z",
          "type": "REVERSAL_AMOUNT",
          "price_guarantee": "FIRM",
          "sender_address": "trans_usd_isendmock@test.cloud.isendmock",
          "receiver_address": "trans_usd_isend@uat.sgp.isend",
          "amount": "121.000000000",
          "currency_code": "USD",
          "currency_code_filter": null,
          "service_type": null,
          "quote_elements": [
            {
              "quote_element_id": "d94691c3-f317-4646-8a15-82fec9ea6e06",
              "quote_element_type": "TRANSFER",
              "quote_element_order": "1",
              "sender_address": "trans_usd_isendmock@test.cloud.isendmock",
              "receiver_address": "conct_usd_isend_isendmock@test.cloud.isendmock",
              "sending_amount": "121.000000000",
              "receiving_amount": "121.000000000",
              "sending_fee": "0.000000000",
              "receiving_fee": "0.000000000",
              "sending_currency_code": null,
              "receiving_currency_code": null,
              "fx_rate": null,
              "transfer_currency_code": "USD"
            },
            {
              "quote_element_id": "55b5cabe-8d27-4945-92ac-18001b7d88e4",
              "quote_element_type": "TRANSFER",
              "quote_element_order": "2",
              "sender_address": "conct_usd_isend_isendmock@uat.sgp.isend",
              "receiver_address": "trans_usd_isend@uat.sgp.isend",
              "sending_amount": "121.000000000",
              "receiving_amount": "121.000000000",
              "sending_fee": "0.000000000",
              "receiving_fee": "0.000000000",
              "sending_currency_code": null,
              "receiving_currency_code": null,
              "fx_rate": null,
              "transfer_currency_code": "USD"
            }
          ],
          "liquidity_warning": null,
          "payment_method": null,
          "payment_method_fields": null,
          "payout_method_info": null
        },
        "fee_info": null
      },
      "ripplenet_info": [],
      "execution_condition": "PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}",
      "crypto_transaction_id": "90842455-a920-4ce4-827b-4a0e7b900999",
      "validator": "test.cloud.isendmock",
      "payment_type": "RETURN",
      "returns_payment_with_id": "9ec07098-6b0d-498a-9300-7c50f81cebec",
      "returned_by_payment_with_id": null,
      "execution_results": [
        {
          "execution_result_id": "d94691c3-f317-4646-8a15-82fec9ea6e06",
          "execution_timestamp": "2024-03-28T05:56:54.173Z",
          "execution_result_type": "TRANSFER",
          "execution_result_order": "1",
          "sender_address": "trans_usd_isendmock@test.cloud.isendmock",
          "receiver_address": "conct_usd_isend_isendmock@test.cloud.isendmock",
          "sending_amount": "121.000000000",
          "receiving_amount": "121.000000000",
          "sending_fee": "0.000000000",
          "receiving_fee": "0.000000000",
          "sending_currency_code": null,
          "receiving_currency_code": null,
          "fx_rate": null,
          "transfer_currency_code": "USD",
          "intermediary_delta": null,
          "incentive_type": null,
          "incentive_value": null,
          "transaction_hash": null,
          "venue_id": null,
          "fiat_adjusted_value": null,
          "odl_payment_id": null
        },
        {
          "execution_result_id": "55b5cabe-8d27-4945-92ac-18001b7d88e4",
          "execution_timestamp": "2024-03-28T05:56:54.056Z",
          "execution_result_type": "TRANSFER",
          "execution_result_order": "2",
          "sender_address": "conct_usd_isend_isendmock@uat.sgp.isend",
          "receiver_address": "trans_usd_isend@uat.sgp.isend",
          "sending_amount": "121.000000000",
          "receiving_amount": "121.000000000",
          "sending_fee": "0.000000000",
          "receiving_fee": "0.000000000",
          "sending_currency_code": null,
          "receiving_currency_code": null,
          "fx_rate": null,
          "transfer_currency_code": "USD",
          "intermediary_delta": null,
          "incentive_type": null,
          "incentive_value": null,
          "transaction_hash": null,
          "venue_id": null,
          "fiat_adjusted_value": null,
          "odl_payment_id": null
        }
      ],
      "liquidation_execution_results": [],
      "liquidation_details": null,
      "push_forward_execution_results": [],
      "direct_payment_id": null,
      "transaction_payment_id": null,
      "accepted_at": "2024-03-28T05:54:23.168Z",
      "locked_at": "2024-03-28T05:55:46.341Z",
      "executed_at": "2024-03-28T05:56:54.386Z",
      "completed_at": "2024-03-28T05:57:10.536Z",
      "returned_at": null,
      "failed_at": null,
      "internal_info": {
        "connector_role": "RECEIVING",
        "labels": [],
        "internal_id": null
      },
      "user_info": [
        {
          "node_address": "test.cloud.isendmock",
          "accepted": [
            {
              "json": {},
              "created_at": "2024-03-28T05:54:22.996Z",
              "subState": ""
            }
          ],
          "locked": [],
          "lock_declined": [],
          "retry_accept": [],
          "retry_settlement": [],
          "settlement": [
            {
              "json": {},
              "created_at": "2024-03-28T05:56:53.230Z",
              "subState": ""
            }
          ],
          "settlement_declined": [],
          "failed": [],
          "executed": [],
          "completed": [],
          "forwarded": [],
          "returned": [
            {
              "json": {
                "code": "NARR",
                "code_detail": "Narrative",
                "description": "Sender Cancel request"
              },
              "created_at": "2024-03-28T05:46:23.912Z",
              "subState": ""
            }
          ],
          "processing_compliance": []
        },
        {
          "node_address": "uat.sgp.isend",
          "accepted": [],
          "locked": [
            {
              "json": {},
              "created_at": "2024-03-28T05:55:46.309Z",
              "subState": ""
            }
          ],
          "lock_declined": [],
          "retry_accept": [],
          "retry_settlement": [],
          "settlement": [],
          "settlement_declined": [],
          "failed": [],
          "executed": [],
          "completed": [
            {
              "json": {},
              "created_at": "2024-03-28T05:57:10.524Z",
              "subState": ""
            }
          ],
          "forwarded": [],
          "returned": [],
          "processing_compliance": []
        }
      ]
    }
  ]
}

Please Help PS: The Logstash code was given to me by ChatGPT and the expected output also from ChatGPT.

dadoonet · April 26, 2024, 6:00am

Welcome!

You need to replace the filter part with a dissect filter instead.

Suman_ISMT · April 26, 2024, 6:04am

Thank you Sir for replying.

Do you mean I have to use the following line:

input {
  cloudwatch_logs {
    log_group => ["/aws/lambda/RippleCheckReturnedTransactionLambda"]
    region => "ap-south-1"
    access_key_id => "*********************"
    secret_access_key => "************************************"
  }
}

filter {
  dissect {
    mapping => {
      "message" => "%{[@metadata][timestamp]} %{[@metadata][log_level]} %{[@metadata][source]}: %{[@metadata][message]}"
    }
    remove_field => ["message"]
  }

  # Flatten the nested JSON structure
  mutate {
    add_field => {
      "[sort][content]" => "%{[sort][content]}"
      "contract" => "%{[contract]}"
    }
    remove_field => ["[sort][content]", "contract"]
  }
}

output {
  elasticsearch {
    index => "awscloudwatch"
    ssl => true
    cacert => "/home/*******/ca_logstash.cer"
    ssl_certificate_verification => true
    hosts => ["https://***.***.**.**:9200"]
    user => "elastic"
    password => "*********"
  }
}

dadoonet · April 26, 2024, 7:21am

Yes. That looks the way to go but you need to split a bit more the dissect filter if you want to generate the JSON you mentioned earlier.

And actually I did not notice it earlier on but it seems that you have a message field which contains both text and json. "the json is: {....}"

So you should first I think remove the json is: from the message field. Ideally do that from the source...

But actually, I also noticed that you should may be not be using Logstash with a non official and not maintained plugin (unless I'm wrong) but instead use this integration: AWS CloudWatch | Documentation

And then create an ingest pipeline to do the extraction you want.

Suman_ISMT · April 26, 2024, 8:01am

Okay thank you for your advice. Since logstash is not working for me can you please tell me more about the aws cloud watch integration I don't have SQS enabled so how can I connect my elastic-integration with awscloudwatch logs?

dadoonet · April 26, 2024, 8:57am

I never done that but I suppose that you could start from AWS CloudWatch | Documentation

There a link to Getting started: Monitor hosts with Elastic Observability | Starting with the Elasticsearch Platform and its Solutions [8.13] | Elastic

You said that the logs are coming from a lambda, right? So may be the best tool in that case is this one: Elastic Serverless Forwarder for AWS | Elastic Serverless Forwarder Guide | Elastic

But again I'm not an expert on that field...

Suman_ISMT · April 26, 2024, 9:17am

I did the above dissect but still I don't see any changes on the kibana dashboard. And I tried adding the elastic-agent with aws cloudwatch using functionbeat. But didn't help.

PS: I am using elasticsearch, kibana, logstash, elastic-agent on v7.14.0.

Sir

dadoonet · April 26, 2024, 9:30am

So let's go back to the basics:

You need to collect logs from your lambda
You need to modify each line of log to have the desired format
You need to display that in Kibana

Let's go step by step. What have you been able to do so far and with which tool?

You should upgrade to at least 7.17. Ideally to 8.13.2.

Suman_ISMT · April 26, 2024, 9:39am

I did tried upgrading the elastic, kibana, logstash but at the end it gave me an error saying node on version 7.14 atleast to be on 7.17 so I reinstalled everything since there is no roll back feature.

So at first I setup the elasticsearch, kibana. Later-on on the remote server I installed logstash and using logstash I am able to connect to aws lamda and visualize the log.

Afterwards, I came to realize that the message string field have some nested json data coming in to the kibana is a single string format from the image we can see that.

So, I was looking for a way to get the string field message in json beautified format because you know it will be easy to troubleshoot the issue.

The data is coming from awslamda -> logstash -> elasticsearch -> kibana

Can you please give me an example, if I want to delete the message string field how would the configuration of logstash.conf look like?? If there is a way to delete the message field then there must be a way I can beautify the nested json, right?

dadoonet · April 26, 2024, 11:18am

If you are starting from scratch, please use 8.13.2. That will ease your life...

Yes but I'm afraid it's not the best way to do this. I mean that you might not need Logstash at all...
But if you still want to go that way, you should share:

the current job configuration
a typical message that can be used to reproduce the problem. Ideally, make it as simple as possible

Suman_ISMT · April 26, 2024, 11:36am

Let's try using 7.14 I already deployed this version now again installing 8.13.2 seems like a tedious task to do.

This is my logstash configuration:

input {
  cloudwatch {
    filters => { "tag:Group" => ["/aws/lambda/RippleCheckReturnedTransactionLambda"] }
    region => "ap-south-1"
    access_key_id => "******************"
    secret_access_key => "*******************************"
  }
}

filter {
  dissect {
    mapping => {
      "message" => "%{[@metadata][timestamp]} %{[@metadata][log_level]} %{[@metadata][source]}: %{[@metadata][message]}"
    }
    remove_field => ["message"]
  }

  # Flatten the nested JSON structure
  mutate {
    add_field => {
      "[sort][content]" => "%{[sort][content]}"
      "contract" => "%{[contract]}"
    }
    remove_field => ["[sort][content]", "contract"]
  }
}

output {
  elasticsearch {
    index => "awscloudwatch"
    ssl => true
    cacert => "/home/*******/ca_logstash.cer"
    ssl_certificate_verification => true
    hosts => ["https://***.***.**.**:9200"]
    user => "elastic"
    password => "*********"
  }
}

I don't know if this message can reproduce the error or not but here is the json message from kibana.

{
  "_index": "awscloudwatch",
  "_type": "_doc",
  "_id": "nYdwE48BfWEk07lFruyT",
  "_score": 1,
  "_ignored": [
    "message.keyword"
  ],
  "_source": {
    "message": "{\"timestamp\":\"2024-04-23T04:53:28.9049309Z\",\"level\":\"Information\",\"service\":\"service_undefined\",\"name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"message\":\"the json is: {\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\",\\\"payment_state\\\":\\\"COMPLETED\\\",\\\"modified_at\\\":\\\"2024-03-28T05:57:10.540Z\\\",\\\"contract\\\":{\\\"sender_end_to_end_id\\\":\\\"IpayPinc10011\\\",\\\"created_at\\\":\\\"2024-03-28T05:54:22.671Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"quote\\\":{\\\"quote_id\\\":\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\",\\\"created_at\\\":\\\"2024-03-28T05:46:23.907Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"type\\\":\\\"REVERSAL_AMOUNT\\\",\\\"price_guarantee\\\":\\\"FIRM\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"amount\\\":\\\"121.000000000\\\",\\\"currency_code\\\":\\\"USD\\\",\\\"currency_code_filter\\\":null,\\\"service_type\\\":null,\\\"quote_elements\\\":[{\\\"quote_element_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"},{\\\"quote_element_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"}],\\\"liquidity_warning\\\":null,\\\"payment_method\\\":null,\\\"payment_method_fields\\\":null,\\\"payout_method_info\\\":null},\\\"fee_info\\\":null},\\\"ripplenet_info\\\":[],\\\"execution_condition\\\":\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\",\\\"crypto_transaction_id\\\":\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\",\\\"validator\\\":\\\"test.cloud.isendmock\\\",\\\"payment_type\\\":\\\"RETURN\\\",\\\"returns_payment_with_id\\\":\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\",\\\"returned_by_payment_with_id\\\":null,\\\"execution_results\\\":[{\\\"execution_result_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.173Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null},{\\\"execution_result_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.056Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null}],\\\"liquidation_execution_results\\\":[],\\\"liquidation_details\\\":null,\\\"push_forward_execution_results\\\":[],\\\"direct_payment_id\\\":null,\\\"transaction_payment_id\\\":null,\\\"accepted_at\\\":\\\"2024-03-28T05:54:23.168Z\\\",\\\"locked_at\\\":\\\"2024-03-28T05:55:46.341Z\\\",\\\"executed_at\\\":\\\"2024-03-28T05:56:54.386Z\\\",\\\"completed_at\\\":\\\"2024-03-28T05:57:10.536Z\\\",\\\"returned_at\\\":null,\\\"failed_at\\\":null,\\\"internal_info\\\":{\\\"connector_role\\\":\\\"RECEIVING\\\",\\\"labels\\\":[],\\\"internal_id\\\":null},\\\"user_info\\\":[{\\\"node_address\\\":\\\"test.cloud.isendmock\\\",\\\"accepted\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:54:22.996Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"locked\\\":[],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:56:53.230Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[],\\\"forwarded\\\":[],\\\"returned\\\":[{\\\"json\\\":{\\\"code\\\":\\\"NARR\\\",\\\"code_detail\\\":\\\"Narrative\\\",\\\"description\\\":\\\"Sender Cancel request\\\"},\\\"created_at\\\":\\\"2024-03-28T05:46:23.912Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"processing_compliance\\\":[]},{\\\"node_address\\\":\\\"uat.sgp.isend\\\",\\\"accepted\\\":[],\\\"locked\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:55:46.309Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:57:10.524Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"forwarded\\\":[],\\\"returned\\\":[],\\\"processing_compliance\\\":[]}]}]}\"}\n",
    "@version": "1",
    "@timestamp": "2024-04-23T04:53:28.905Z",
    "cloudwatch_logs": {
      "log_group": "/aws/lambda/RippleCheckReturnedTransactionLambda",
      "ingestion_time": "2024-04-23T04:53:37.020Z",
      "event_id": "38220087755599447404861821884188269337510756578728804356",
      "log_stream": "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    }
  },
  "fields": {
    "@timestamp": [
      "2024-04-23T04:53:28.905Z"
    ],
    "cloudwatch_logs.event_id": [
      "38220087755599447404861821884188269337510756578728804356"
    ],
    "cloudwatch_logs.event_id.keyword": [
      "38220087755599447404861821884188269337510756578728804356"
    ],
    "cloudwatch_logs.log_group": [
      "/aws/lambda/RippleCheckReturnedTransactionLambda"
    ],
    "@version": [
      "1"
    ],
    "cloudwatch_logs.log_stream": [
      "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    ],
    "cloudwatch_logs.log_group.keyword": [
      "/aws/lambda/RippleCheckReturnedTransactionLambda"
    ],
    "cloudwatch_logs.ingestion_time": [
      "2024-04-23T04:53:37.020Z"
    ],
    "@version.keyword": [
      "1"
    ],
    "cloudwatch_logs.log_stream.keyword": [
      "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    ],
    "message": [
      "{\"timestamp\":\"2024-04-23T04:53:28.9049309Z\",\"level\":\"Information\",\"service\":\"service_undefined\",\"name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"message\":\"the json is: {\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\",\\\"payment_state\\\":\\\"COMPLETED\\\",\\\"modified_at\\\":\\\"2024-03-28T05:57:10.540Z\\\",\\\"contract\\\":{\\\"sender_end_to_end_id\\\":\\\"IpayPinc10011\\\",\\\"created_at\\\":\\\"2024-03-28T05:54:22.671Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"quote\\\":{\\\"quote_id\\\":\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\",\\\"created_at\\\":\\\"2024-03-28T05:46:23.907Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"type\\\":\\\"REVERSAL_AMOUNT\\\",\\\"price_guarantee\\\":\\\"FIRM\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"amount\\\":\\\"121.000000000\\\",\\\"currency_code\\\":\\\"USD\\\",\\\"currency_code_filter\\\":null,\\\"service_type\\\":null,\\\"quote_elements\\\":[{\\\"quote_element_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"},{\\\"quote_element_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"}],\\\"liquidity_warning\\\":null,\\\"payment_method\\\":null,\\\"payment_method_fields\\\":null,\\\"payout_method_info\\\":null},\\\"fee_info\\\":null},\\\"ripplenet_info\\\":[],\\\"execution_condition\\\":\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\",\\\"crypto_transaction_id\\\":\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\",\\\"validator\\\":\\\"test.cloud.isendmock\\\",\\\"payment_type\\\":\\\"RETURN\\\",\\\"returns_payment_with_id\\\":\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\",\\\"returned_by_payment_with_id\\\":null,\\\"execution_results\\\":[{\\\"execution_result_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.173Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null},{\\\"execution_result_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.056Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null}],\\\"liquidation_execution_results\\\":[],\\\"liquidation_details\\\":null,\\\"push_forward_execution_results\\\":[],\\\"direct_payment_id\\\":null,\\\"transaction_payment_id\\\":null,\\\"accepted_at\\\":\\\"2024-03-28T05:54:23.168Z\\\",\\\"locked_at\\\":\\\"2024-03-28T05:55:46.341Z\\\",\\\"executed_at\\\":\\\"2024-03-28T05:56:54.386Z\\\",\\\"completed_at\\\":\\\"2024-03-28T05:57:10.536Z\\\",\\\"returned_at\\\":null,\\\"failed_at\\\":null,\\\"internal_info\\\":{\\\"connector_role\\\":\\\"RECEIVING\\\",\\\"labels\\\":[],\\\"internal_id\\\":null},\\\"user_info\\\":[{\\\"node_address\\\":\\\"test.cloud.isendmock\\\",\\\"accepted\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:54:22.996Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"locked\\\":[],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:56:53.230Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[],\\\"forwarded\\\":[],\\\"returned\\\":[{\\\"json\\\":{\\\"code\\\":\\\"NARR\\\",\\\"code_detail\\\":\\\"Narrative\\\",\\\"description\\\":\\\"Sender Cancel request\\\"},\\\"created_at\\\":\\\"2024-03-28T05:46:23.912Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"processing_compliance\\\":[]},{\\\"node_address\\\":\\\"uat.sgp.isend\\\",\\\"accepted\\\":[],\\\"locked\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:55:46.309Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:57:10.524Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"forwarded\\\":[],\\\"returned\\\":[],\\\"processing_compliance\\\":[]}]}]}\"}\n"
    ]
  }
}

What I want to accomplish is the last message field I want that also be beautified as other fields. I am open to any suggestion you give me besides using an aws configured elastic-agent integration because the protocol won't allow me to do so.

So the only option is using log aggrerator like logstash or any other you would suggest That would be highly appreciated sir.

dadoonet · April 26, 2024, 2:28pm

I'd probably use the JSON filter plugin | Logstash Reference [8.13] | Elastic on the message field. The json contains a message field as well. Not sure on how this will behave by default, so I'd probably use the target option.

It should be something like:

filter {
  json {
    source => "message"
    target => "doc"
  }

  json {
    source => "[doc][message]"
    target => "log"
  }
}

The problem is that your message inner field contains "message":"the json is: {\"foo\": \"bar\"}". Which is not a proper json.

So either you remove from the source "the json is:" or you find a way to remove it later in Logstash. May be a dissect like this will do it:

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}

Not tested but this gives an idea on what you could try.

My advice: be iterative. Start from the first filter. If it gives what you are expecting, then move to the second filter...

No need to send data to Elasticsearch yet. Use the stdout output plugin instead. That will make your life easier until the filter part is properly done.

Suman_ISMT · April 29, 2024, 9:47am

Thank you for your help I will look at it once again and come back to you.

Suman_ISMT · April 29, 2024, 10:01am

Logstash Configuration:

input {
  cloudwatch {
    filters => { "tag:Group" => ["/aws/lambda/RippleCheckReturnedTransactionLambda"] }
    region => "ap-south-1"
    access_key_id => "************************"
    secret_access_key => "*****************************"
    enable_metric => false
  }
}

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}
output {
 stdout{ codec => json}
}

Running logstash with the above configuration:

root@fintech:/usr/share/logstash# bin/logstash -f /etc/logstash/conf.d/logstash.conf
Using bundled JDK: /usr/share/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2024-04-29 09:54:34.046 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +indy +jit [linux-x86_64]"}
[WARN ] 2024-04-29 09:54:35.784 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2024-04-29 09:54:47.159 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
[INFO ] 2024-04-29 09:54:49.091 [Converge PipelineAction::Create<main>] Reflections - Reflections took 188 ms to scan 1 urls, producing 120 keys and 417 values 
[WARN ] 2024-04-29 09:54:54.161 [[main]-pipeline-manager] json - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[INFO ] 2024-04-29 09:54:54.434 [[main]-pipeline-manager] javapipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/etc/logstash/conf.d/logstash.conf"], :thread=>"#<Thread:0x705bf517 run>"}
[INFO ] 2024-04-29 09:55:04.608 [[main]-pipeline-manager] javapipeline - Pipeline Java execution initialization time {"seconds"=>10.17}
[INFO ] 2024-04-29 09:55:04.634 [[main]-pipeline-manager] javapipeline - Pipeline started {"pipeline.id"=>"main"}
[INFO ] 2024-04-29 09:55:04.767 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[INFO ] 2024-04-29 09:55:04.826 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
Version 2 of the Ruby SDK will enter maintenance mode as of November 20, 2020. To continue receiving service updates and new features, please upgrade to Version 3. More information can be found here: https://aws.amazon.com/blogs/developer/deprecation-schedule-for-aws-sdk-for-ruby-v2/
[INFO ] 2024-04-29 09:55:24.682 [[main]<cloudwatch] cloudwatch - [Aws::CloudWatch::Client 200 14.164231 0 retries] list_metrics(namespace:"AWS/EC2")  

[ERROR] 2024-04-29 09:55:24.707 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"***********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:25.723 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:25.725 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*******************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:26.726 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:26.727 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"***************************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:27.728 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:27.730 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:28.731 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:28.732 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*****************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:29.733 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:29.734 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*****************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:30.735 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:30.737 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"***********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:31.738 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:31.755 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"***********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
^C[INFO ] 2024-04-29 09:55:32.756 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:32.758 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"**********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[WARN ] 2024-04-29 09:55:32.810 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2024-04-29 09:55:33.879 [[main]-pipeline-manager] javapipeline - Pipeline terminated {"pipeline.id"=>"main"}
[INFO ] 2024-04-29 09:55:33.959 [LogStash::Runner] runner - Logstash shut down.

PS: The recently used access key and secret does not have access to current lamda logs only upto 23rd April.

Also, could you also help me sort this new error.

Suman_ISMT · April 29, 2024, 10:08am

Logstash Configuration:

input {
  cloudwatch {
    filters => { "tag:Group" => ["/aws/lambda/RippleCheckReturnedTransactionLambda"] }
    region => "ap-south-1"
    access_key_id => "***********************"
    secret_access_key => "*************************"
    enable_metric => false
  }
}

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}
output {
 stdout{ codec => json}
}

I have encounter another error.

root@fintech:/usr/share/logstash# bin/logstash -f /etc/logstash/conf.d/logstash.conf
Using bundled JDK: /usr/share/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2024-04-29 09:54:34.046 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +indy +jit [linux-x86_64]"}
[WARN ] 2024-04-29 09:54:35.784 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2024-04-29 09:54:47.159 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
[INFO ] 2024-04-29 09:54:49.091 [Converge PipelineAction::Create<main>] Reflections - Reflections took 188 ms to scan 1 urls, producing 120 keys and 417 values 
[WARN ] 2024-04-29 09:54:54.161 [[main]-pipeline-manager] json - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[INFO ] 2024-04-29 09:54:54.434 [[main]-pipeline-manager] javapipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/etc/logstash/conf.d/logstash.conf"], :thread=>"#<Thread:0x705bf517 run>"}
[INFO ] 2024-04-29 09:55:04.608 [[main]-pipeline-manager] javapipeline - Pipeline Java execution initialization time {"seconds"=>10.17}
[INFO ] 2024-04-29 09:55:04.634 [[main]-pipeline-manager] javapipeline - Pipeline started {"pipeline.id"=>"main"}
[INFO ] 2024-04-29 09:55:04.767 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[INFO ] 2024-04-29 09:55:04.826 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
Version 2 of the Ruby SDK will enter maintenance mode as of November 20, 2020. To continue receiving service updates and new features, please upgrade to Version 3. More information can be found here: https://aws.amazon.com/blogs/developer/deprecation-schedule-for-aws-sdk-for-ruby-v2/
[INFO ] 2024-04-29 09:55:24.682 [[main]<cloudwatch] cloudwatch - [Aws::CloudWatch::Client 200 14.164231 0 retries] list_metrics(namespace:"AWS/EC2")  

[ERROR] 2024-04-29 09:55:24.707 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*****************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:25.723 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:25.725 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:26.726 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:26.727 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"**********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:27.728 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:27.730 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"**********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:28.731 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:28.732 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"**********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:29.733 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:29.734 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"************************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:30.735 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:30.737 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[INFO ] 2024-04-29 09:55:31.738 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:31.755 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"*****************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
^C[INFO ] 2024-04-29 09:55:32.756 [[main]<cloudwatch] cloudwatch - Polling CloudWatch API
[ERROR] 2024-04-29 09:55:32.758 [[main]<cloudwatch] javapipeline - A plugin had an unrecoverable error. Will restart this plugin.
  Pipeline_id:main
  Plugin: <LogStash::Inputs::CloudWatch access_key_id=>"**********************", secret_access_key=><password>, filters=>{"tag:Group"=>["/aws/lambda/RippleCheckReturnedTransactionLambda"]}, enable_metric=>false, id=>"1da43c15a460c62656dae8e32de5f28171332759e1ed5cd88e1b5f4a420cd26b", region=>"ap-south-1", codec=><LogStash::Codecs::Plain id=>"plain_f7b95423-13f2-453f-b8c7-f60d9f6513cd", enable_metric=>true, charset=>"UTF-8">, role_session_name=>"logstash", namespace=>"AWS/EC2", metrics=>["CPUUtilization", "DiskReadOps", "DiskWriteOps", "NetworkIn", "NetworkOut"], statistics=>["SampleCount", "Average", "Minimum", "Maximum", "Sum"], interval=>900, period=>300, combined=>false>
  Error: No metrics to query
  Exception: RuntimeError
  Stack: /usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-input-cloudwatch-2.2.4/lib/logstash/inputs/cloudwatch.rb:154:in `run'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:405:in `inputworker'
/usr/share/logstash/logstash-core/lib/logstash/java_pipeline.rb:396:in `block in start_input'
[WARN ] 2024-04-29 09:55:32.810 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2024-04-29 09:55:33.879 [[main]-pipeline-manager] javapipeline - Pipeline terminated {"pipeline.id"=>"main"}
[INFO ] 2024-04-29 09:55:33.959 [LogStash::Runner] runner - Logstash shut down.

dadoonet · April 29, 2024, 2:40pm

From what I can see from the logs, it's an error retrieving the data from cloudwatch.
I'd try with something even more easy using a stdin input plugin. And then copy paste a typical logline in the console. So you can see the effect without cloudwatch plugin.

Suman_ISMT · April 30, 2024, 3:35am

Do you mean to say that in standard input I can place the portion of the log and see it's beautified output?

If yes, I would need some help doing that. Can you help me on that sir?

Suman_ISMT · April 30, 2024, 3:49am

I think I did what you wanted me do. But I don't see output anywhere do I need my kibana instance running?

Logstash Configuration:

input {
 file {
   path => "/var/log/aws-cloudwatch.log"
 }
 stdin { codec => json }
}

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}
output {
 stdout{ codec => json}
}

Output of running the above configuration:

root@fintech:/usr/share/logstash# bin/logstash -f /etc/logstash/conf.d/logstash.conf
Using bundled JDK: /usr/share/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2024-04-30 03:44:54.734 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +indy +jit [linux-x86_64]"}
[WARN ] 2024-04-30 03:44:55.254 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2024-04-30 03:44:56.719 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
[INFO ] 2024-04-30 03:44:57.282 [Converge PipelineAction::Create<main>] Reflections - Reflections took 90 ms to scan 1 urls, producing 120 keys and 417 values 
[WARN ] 2024-04-30 03:44:58.197 [Converge PipelineAction::Create<main>] file - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[WARN ] 2024-04-30 03:44:58.271 [Converge PipelineAction::Create<main>] stdin - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[WARN ] 2024-04-30 03:44:58.476 [[main]-pipeline-manager] json - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[INFO ] 2024-04-30 03:44:58.534 [[main]-pipeline-manager] javapipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/etc/logstash/conf.d/logstash.conf"], :thread=>"#<Thread:0x73f59b06 run>"}
[INFO ] 2024-04-30 03:45:00.911 [[main]-pipeline-manager] javapipeline - Pipeline Java execution initialization time {"seconds"=>2.37}
[INFO ] 2024-04-30 03:45:01.179 [[main]-pipeline-manager] file - No sincedb_path set, generating one based on the "path" setting {:sincedb_path=>"/usr/share/logstash/data/plugins/inputs/file/.sincedb_1adcce4500bcb3d3c1aa55b91ef3bb97", :path=>["/var/log/aws-cloudwatch.log"]}
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by com.jrubystdinchannel.StdinChannelLibrary$Reader (file:/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/jruby-stdin-channel-0.2.0-java/lib/jruby_stdin_channel/jruby_stdin_channel.jar) to field java.io.FilterInputStream.in
WARNING: Please consider reporting this to the maintainers of com.jrubystdinchannel.StdinChannelLibrary$Reader
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
[INFO ] 2024-04-30 03:45:01.580 [[main]-pipeline-manager] stdin - Automatically switching from json to json_lines codec {:plugin=>"stdin"}
[INFO ] 2024-04-30 03:45:01.714 [[main]-pipeline-manager] javapipeline - Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[INFO ] 2024-04-30 03:45:02.672 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[INFO ] 2024-04-30 03:45:02.855 [[main]<file] observingtail - START, creating Discoverer, Watch with file and sincedb collections

^C[WARN ] 2024-04-30 03:46:08.287 [SIGINT handler] runner - SIGINT received. Shutting down.
[INFO ] 2024-04-30 03:46:08.333 [Converge PipelineAction::Stop<main>] observingtail - QUIT - closing all files and shutting down.
[INFO ] 2024-04-30 03:46:09.217 [[main]-pipeline-manager] javapipeline - Pipeline terminated {"pipeline.id"=>"main"}
[INFO ] 2024-04-30 03:46:09.510 [LogStash::Runner] runner - Logstash shut down.

Log file on /var/log/aws-cloudwatch.log:

root@fintech:/usr/share/logstash# cat /var/log/aws-cloudwatch.log 
{
  "_index": "awscloudwatch",
  "_type": "_doc",
  "_id": "nYdwE48BfWEk07lFruyT",
  "_score": 1,
  "_ignored": [
    "message.keyword"
  ],
  "_source": {
    "message": "{\"timestamp\":\"2024-04-23T04:53:28.9049309Z\",\"level\":\"Information\",\"service\":\"service_undefined\",\"name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"message\":\"the json is: {\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\",\\\"payment_state\\\":\\\"COMPLETED\\\",\\\"modified_at\\\":\\\"2024-03-28T05:57:10.540Z\\\",\\\"contract\\\":{\\\"sender_end_to_end_id\\\":\\\"IpayPinc10011\\\",\\\"created_at\\\":\\\"2024-03-28T05:54:22.671Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"quote\\\":{\\\"quote_id\\\":\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\",\\\"created_at\\\":\\\"2024-03-28T05:46:23.907Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"type\\\":\\\"REVERSAL_AMOUNT\\\",\\\"price_guarantee\\\":\\\"FIRM\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"amount\\\":\\\"121.000000000\\\",\\\"currency_code\\\":\\\"USD\\\",\\\"currency_code_filter\\\":null,\\\"service_type\\\":null,\\\"quote_elements\\\":[{\\\"quote_element_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"},{\\\"quote_element_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"}],\\\"liquidity_warning\\\":null,\\\"payment_method\\\":null,\\\"payment_method_fields\\\":null,\\\"payout_method_info\\\":null},\\\"fee_info\\\":null},\\\"ripplenet_info\\\":[],\\\"execution_condition\\\":\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\",\\\"crypto_transaction_id\\\":\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\",\\\"validator\\\":\\\"test.cloud.isendmock\\\",\\\"payment_type\\\":\\\"RETURN\\\",\\\"returns_payment_with_id\\\":\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\",\\\"returned_by_payment_with_id\\\":null,\\\"execution_results\\\":[{\\\"execution_result_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.173Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null},{\\\"execution_result_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.056Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null}],\\\"liquidation_execution_results\\\":[],\\\"liquidation_details\\\":null,\\\"push_forward_execution_results\\\":[],\\\"direct_payment_id\\\":null,\\\"transaction_payment_id\\\":null,\\\"accepted_at\\\":\\\"2024-03-28T05:54:23.168Z\\\",\\\"locked_at\\\":\\\"2024-03-28T05:55:46.341Z\\\",\\\"executed_at\\\":\\\"2024-03-28T05:56:54.386Z\\\",\\\"completed_at\\\":\\\"2024-03-28T05:57:10.536Z\\\",\\\"returned_at\\\":null,\\\"failed_at\\\":null,\\\"internal_info\\\":{\\\"connector_role\\\":\\\"RECEIVING\\\",\\\"labels\\\":[],\\\"internal_id\\\":null},\\\"user_info\\\":[{\\\"node_address\\\":\\\"test.cloud.isendmock\\\",\\\"accepted\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:54:22.996Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"locked\\\":[],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:56:53.230Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[],\\\"forwarded\\\":[],\\\"returned\\\":[{\\\"json\\\":{\\\"code\\\":\\\"NARR\\\",\\\"code_detail\\\":\\\"Narrative\\\",\\\"description\\\":\\\"Sender Cancel request\\\"},\\\"created_at\\\":\\\"2024-03-28T05:46:23.912Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"processing_compliance\\\":[]},{\\\"node_address\\\":\\\"uat.sgp.isend\\\",\\\"accepted\\\":[],\\\"locked\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:55:46.309Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:57:10.524Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"forwarded\\\":[],\\\"returned\\\":[],\\\"processing_compliance\\\":[]}]}]}\"}\n",
    "@version": "1",
    "@timestamp": "2024-04-23T04:53:28.905Z",
    "cloudwatch_logs": {
      "log_group": "/aws/lambda/RippleCheckReturnedTransactionLambda",
      "ingestion_time": "2024-04-23T04:53:37.020Z",
      "event_id": "38220087755599447404861821884188269337510756578728804356",
      "log_stream": "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    }
  },
  "fields": {
    "@timestamp": [
      "2024-04-23T04:53:28.905Z"
    ],
    "cloudwatch_logs.event_id": [
      "38220087755599447404861821884188269337510756578728804356"
    ],
    "cloudwatch_logs.event_id.keyword": [
      "38220087755599447404861821884188269337510756578728804356"
    ],
    "cloudwatch_logs.log_group": [
      "/aws/lambda/RippleCheckReturnedTransactionLambda"
    ],
    "@version": [
      "1"
    ],
    "cloudwatch_logs.log_stream": [
      "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    ],
    "cloudwatch_logs.log_group.keyword": [
      "/aws/lambda/RippleCheckReturnedTransactionLambda"
    ],
    "cloudwatch_logs.ingestion_time": [
      "2024-04-23T04:53:37.020Z"
    ],
    "@version.keyword": [
      "1"
    ],
    "cloudwatch_logs.log_stream.keyword": [
      "2024/04/23/[$LATEST]4879555d8e39419794f8b7ef89db9021"
    ],
    "message": [
      "{\"timestamp\":\"2024-04-23T04:53:28.9049309Z\",\"level\":\"Information\",\"service\":\"service_undefined\",\"name\":\"AWS.Lambda.Powertools.Logging.Logger\",\"message\":\"the json is: {\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\",\\\"payment_state\\\":\\\"COMPLETED\\\",\\\"modified_at\\\":\\\"2024-03-28T05:57:10.540Z\\\",\\\"contract\\\":{\\\"sender_end_to_end_id\\\":\\\"IpayPinc10011\\\",\\\"created_at\\\":\\\"2024-03-28T05:54:22.671Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"quote\\\":{\\\"quote_id\\\":\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\",\\\"created_at\\\":\\\"2024-03-28T05:46:23.907Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"type\\\":\\\"REVERSAL_AMOUNT\\\",\\\"price_guarantee\\\":\\\"FIRM\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"amount\\\":\\\"121.000000000\\\",\\\"currency_code\\\":\\\"USD\\\",\\\"currency_code_filter\\\":null,\\\"service_type\\\":null,\\\"quote_elements\\\":[{\\\"quote_element_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"},{\\\"quote_element_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"}],\\\"liquidity_warning\\\":null,\\\"payment_method\\\":null,\\\"payment_method_fields\\\":null,\\\"payout_method_info\\\":null},\\\"fee_info\\\":null},\\\"ripplenet_info\\\":[],\\\"execution_condition\\\":\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\",\\\"crypto_transaction_id\\\":\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\",\\\"validator\\\":\\\"test.cloud.isendmock\\\",\\\"payment_type\\\":\\\"RETURN\\\",\\\"returns_payment_with_id\\\":\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\",\\\"returned_by_payment_with_id\\\":null,\\\"execution_results\\\":[{\\\"execution_result_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.173Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null},{\\\"execution_result_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.056Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null}],\\\"liquidation_execution_results\\\":[],\\\"liquidation_details\\\":null,\\\"push_forward_execution_results\\\":[],\\\"direct_payment_id\\\":null,\\\"transaction_payment_id\\\":null,\\\"accepted_at\\\":\\\"2024-03-28T05:54:23.168Z\\\",\\\"locked_at\\\":\\\"2024-03-28T05:55:46.341Z\\\",\\\"executed_at\\\":\\\"2024-03-28T05:56:54.386Z\\\",\\\"completed_at\\\":\\\"2024-03-28T05:57:10.536Z\\\",\\\"returned_at\\\":null,\\\"failed_at\\\":null,\\\"internal_info\\\":{\\\"connector_role\\\":\\\"RECEIVING\\\",\\\"labels\\\":[],\\\"internal_id\\\":null},\\\"user_info\\\":[{\\\"node_address\\\":\\\"test.cloud.isendmock\\\",\\\"accepted\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:54:22.996Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"locked\\\":[],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:56:53.230Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[],\\\"forwarded\\\":[],\\\"returned\\\":[{\\\"json\\\":{\\\"code\\\":\\\"NARR\\\",\\\"code_detail\\\":\\\"Narrative\\\",\\\"description\\\":\\\"Sender Cancel request\\\"},\\\"created_at\\\":\\\"2024-03-28T05:46:23.912Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"processing_compliance\\\":[]},{\\\"node_address\\\":\\\"uat.sgp.isend\\\",\\\"accepted\\\":[],\\\"locked\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:55:46.309Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:57:10.524Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"forwarded\\\":[],\\\"returned\\\":[],\\\"processing_compliance\\\":[]}]}]}\"}\n"
    ]
  }
}

dadoonet · April 30, 2024, 11:35am

Let's simplify a bit more:

input {
 stdin { codec => json }
}

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}
output {
 stdout{ codec => json}
}

And then copy paste a typical line of log. DO NOT PASTE THE CONTENT FROM ELASTICSEARCH. According to your previous response, it's probably something like:

{"timestamp":"2024-04-23T04:53:28.9049309Z","level":"Information","service":"service_undefined","name":"AWS.Lambda.Powertools.Logging.Logger","message":"the json is: {\"foo\":\"bar\"}"}

Suman_ISMT · April 30, 2024, 11:45am

root@fintech:/usr/share/logstash# bin/logstash -f /etc/logstash/conf.d/logstash.conf
Using bundled JDK: /usr/share/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/bundler-1.17.3/lib/bundler/rubygems_integration.rb:200: warning: constant Gem::ConfigMap is deprecated
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
[INFO ] 2024-04-30 11:40:53.313 [main] runner - Starting Logstash {"logstash.version"=>"7.14.0", "jruby.version"=>"jruby 9.2.19.0 (2.5.8) 2021-06-15 55810c552b OpenJDK 64-Bit Server VM 11.0.11+9 on 11.0.11+9 +indy +jit [linux-x86_64]"}
[WARN ] 2024-04-30 11:40:53.882 [LogStash::Runner] multilocal - Ignoring the 'pipelines.yml' file because modules or command line options are specified
[INFO ] 2024-04-30 11:40:57.844 [Api Webserver] agent - Successfully started Logstash API endpoint {:port=>9601}
[INFO ] 2024-04-30 11:40:58.896 [Converge PipelineAction::Create<main>] Reflections - Reflections took 184 ms to scan 1 urls, producing 120 keys and 417 values 
[WARN ] 2024-04-30 11:41:04.236 [Converge PipelineAction::Create<main>] stdin - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[WARN ] 2024-04-30 11:41:05.591 [[main]-pipeline-manager] json - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
[INFO ] 2024-04-30 11:41:05.732 [[main]-pipeline-manager] javapipeline - Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/etc/logstash/conf.d/logstash.conf"], :thread=>"#<Thread:0x3579feb2 run>"}
[INFO ] 2024-04-30 11:41:08.522 [[main]-pipeline-manager] javapipeline - Pipeline Java execution initialization time {"seconds"=>2.79}
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by com.jrubystdinchannel.StdinChannelLibrary$Reader (file:/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/jruby-stdin-channel-0.2.0-java/lib/jruby_stdin_channel/jruby_stdin_channel.jar) to field java.io.FilterInputStream.in
WARNING: Please consider reporting this to the maintainers of com.jrubystdinchannel.StdinChannelLibrary$Reader
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
[INFO ] 2024-04-30 11:41:08.605 [[main]-pipeline-manager] stdin - Automatically switching from json to json_lines codec {:plugin=>"stdin"}
[INFO ] 2024-04-30 11:41:08.632 [[main]-pipeline-manager] javapipeline - Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[INFO ] 2024-04-30 11:41:08.873 [Agent thread] agent - Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\",\\\"payment_state\\\":\\\"COMPLETED\\\",\\\"modified_at\\\":\\\"2024-03-28T05:57:10.540Z\\\",\\\"contract\\\":{\\\"sender_end_to_end_id\\\":\\\"IpayPinc10011\\\",\\\"created_at\\\":\\\"2024-03-28T05:54:22.671Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"quote\\\":{\\\"quote_id\\\":\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\",\\\"created_at\\\":\\\"2024-03-28T05:46:23.907Z\\\",\\\"expires_at\\\":\\\"2024-03-28T06:46:23.907Z\\\",\\\"type\\\":\\\"REVERSAL_AMOUNT\\\",\\\"price_guarantee\\\":\\\"FIRM\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"amount\\\":\\\"121.000000000\\\",\\\"currency_code\\\":\\\"USD\\\",\\\"currency_code_filter\\\":null,\\\"service_type\\\":null,\\\"quote_elements\\\":[{\\\"quote_element_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"},{\\\"quote_element_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"quote_element_type\\\":\\\"TRANSFER\\\",\\\"quote_element_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\"}],\\\"liquidity_warning\\\":null,\\\"payment_method\\\":null,\\\"payment_method_fields\\\":null,\\\"payout_method_info\\\":null},\\\"fee_info\\\":null},\\\"ripplenet_info\\\":[],\\\"execution_condition\\\":\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\",\\\"crypto_transaction_id\\\":\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\",\\\"validator\\\":\\\"test.cloud.isendmock\\\",\\\"payment_type\\\":\\\"RETURN\\\",\\\"returns_payment_with_id\\\":\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\",\\\"returned_by_payment_with_id\\\":null,\\\"execution_results\\\":[{\\\"execution_result_id\\\":\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.173Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"1\\\",\\\"sender_address\\\":\\\"trans_usd_isendmock@test.cloud.isendmock\\\",\\\"receiver_address\\\":\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null},{\\\"execution_result_id\\\":\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\",\\\"execution_timestamp\\\":\\\"2024-03-28T05:56:54.056Z\\\",\\\"execution_result_type\\\":\\\"TRANSFER\\\",\\\"execution_result_order\\\":\\\"2\\\",\\\"sender_address\\\":\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\",\\\"receiver_address\\\":\\\"trans_usd_isend@uat.sgp.isend\\\",\\\"sending_amount\\\":\\\"121.000000000\\\",\\\"receiving_amount\\\":\\\"121.000000000\\\",\\\"sending_fee\\\":\\\"0.000000000\\\",\\\"receiving_fee\\\":\\\"0.000000000\\\",\\\"sending_currency_code\\\":null,\\\"receiving_currency_code\\\":null,\\\"fx_rate\\\":null,\\\"transfer_currency_code\\\":\\\"USD\\\",\\\"intermediary_delta\\\":null,\\\"incentive_type\\\":null,\\\"incentive_value\\\":null,\\\"transaction_hash\\\":null,\\\"venue_id\\\":null,\\\"fiat_adjusted_value\\\":null,\\\"odl_payment_id\\\":null}],\\\"liquidation_execution_results\\\":[],\\\"liquidation_details\\\":null,\\\"push_forward_execution_results\\\":[],\\\"direct_payment_id\\\":null,\\\"transaction_payment_id\\\":null,\\\"accepted_at\\\":\\\"2024-03-28T05:54:23.168Z\\\",\\\"locked_at\\\":\\\"2024-03-28T05:55:46.341Z\\\",\\\"executed_at\\\":\\\"2024-03-28T05:56:54.386Z\\\",\\\"completed_at\\\":\\\"2024-03-28T05:57:10.536Z\\\",\\\"returned_at\\\":null,\\\"failed_at\\\":null,\\\"internal_info\\\":{\\\"connector_role\\\":\\\"RECEIVING\\\",\\\"labels\\\":[],\\\"internal_id\\\":null},\\\"user_info\\\":[{\\\"node_address\\\":\\\"test.cloud.isendmock\\\",\\\"accepted\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:54:22.996Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"locked\\\":[],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:56:53.230Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[],\\\"forwarded\\\":[],\\\"returned\\\":[{\\\"json\\\":{\\\"code\\\":\\\"NARR\\\",\\\"code_detail\\\":\\\"Narrative\\\",\\\"description\\\":\\\"Sender Cancel request\\\"},\\\"created_at\\\":\\\"2024-03-28T05:46:23.912Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"processing_compliance\\\":[]},{\\\"node_address\\\":\\\"uat.sgp.isend\\\",\\\"accepted\\\":[],\\\"locked\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:55:46.309Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"lock_declined\\\":[],\\\"retry_accept\\\":[],\\\"retry_settlement\\\":[],\\\"settlement\\\":[],\\\"settlement_declined\\\":[],\\\"failed\\\":[],\\\"executed\\\":[],\\\"completed\\\":[{\\\"json\\\":{},\\\"created_at\\\":\\\"2024-03-28T05:57:10.524Z\\\",\\\"subState\\\":\\\"\\\"}],\\\"forwarded\\\":[],\\\"returned\\\":[],\\\"processing_compliance\\\":[]}]}]}\
[WARN ] 2024-04-30 11:41:37.192 [[main]<stdin] jsonlines - JSON parse error, original data now in message field {:error=>#<LogStash::Json::ParserError: Unexpected character ('\' (code 92)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')
 at [Source: (String)"\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b48158"[truncated 3595 chars]; line: 1, column: 2]>, :data=>"\\\\\\\"first\\\\\\\":true,\\\\\\\"last\\\\\\\":true,\\\\\\\"number\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"numberOfElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"size\\\\\\\":\\\\\\\"9\\\\\\\",\\\\\\\"totalElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"totalPages\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sort\\\\\\\":[{\\\\\\\"direction\\\\\\\":\\\\\\\"DESC\\\\\\\",\\\\\\\"property\\\\\\\":\\\\\\\"MODIFIED_AT\\\\\\\",\\\\\\\"ignoreCase\\\\\\\":false,\\\\\\\"nullHandling\\\\\\\":\\\\\\\"NATIVE\\\\\\\",\\\\\\\"ascending\\\\\\\":false,\\\\\\\"descending\\\\\\\":true}],\\\\\\\"content\\\\\\\":[{\\\\\\\"payment_id\\\\\\\":\\\\\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\\\\\",\\\\\\\"contract_hash\\\\\\\":\\\\\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\\\\\",\\\\\\\"payment_state\\\\\\\":\\\\\\\"COMPLETED\\\\\\\",\\\\\\\"modified_at\\\\\\\":\\\\\\\"2024-03-28T05:57:10.540Z\\\\\\\",\\\\\\\"contract\\\\\\\":{\\\\\\\"sender_end_to_end_id\\\\\\\":\\\\\\\"IpayPinc10011\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:54:22.671Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"quote\\\\\\\":{\\\\\\\"quote_id\\\\\\\":\\\\\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:46:23.907Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"type\\\\\\\":\\\\\\\"REVERSAL_AMOUNT\\\\\\\",\\\\\\\"price_guarantee\\\\\\\":\\\\\\\"FIRM\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"currency_code_filter\\\\\\\":null,\\\\\\\"service_type\\\\\\\":null,\\\\\\\"quote_elements\\\\\\\":[{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"},{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"2\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"}],\\\\\\\"liquidity_warning\\\\\\\":null,\\\\\\\"payment_method\\\\\\\":null,\\\\\\\"payment_method_fields\\\\\\\":null,\\\\\\\"payout_method_info\\\\\\\":null},\\\\\\\"fee_info\\\\\\\":null},\\\\\\\"ripplenet_info\\\\\\\":[],\\\\\\\"execution_condition\\\\\\\":\\\\\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\\\\\",\\\\\\\"crypto_transaction_id\\\\\\\":\\\\\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\\\\\",\\\\\\\"validator\\\\\\\":\\\\\\\"test.cloud.isendmock\\\\\\\",\\\\\\\"payment_type\\\\\\\":\\\\\\\"RETURN\\\\\\\",\\\\\\\"returns_payment_with_id\\\\\\\":\\\\\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\\\\\",\\\\\\\"returned_by_payment_with_id\\\\\\\":null,\\\\\\\"execution_results\\\\\\\":[{\\\\\\\"execution_result_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"execution_timestamp\\\\\\\":\\\\\\\"2024-03-28T05:56:54.173Z\\\\\\\",\\\\\\\"execution_result_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"execution_result_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"intermediary_delta\\\\\\\":null,\\\\\\\"incentive_type\\\\\\\":null,\\\\\\\"incentive_value\\\\\\\":null,\\\\\\\"transaction_hash\\\\\\\":null,\\\\\\\"venue_id\\\\\\\":null,\\\\\\\"fiat_adjusted_value\\\\\\\":null,\\\\"}
[WARN ] 2024-04-30 11:41:37.462 [[main]>worker1] json - Error parsing json {:source=>"message", :raw=>"\\\\\\\"first\\\\\\\":true,\\\\\\\"last\\\\\\\":true,\\\\\\\"number\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"numberOfElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"size\\\\\\\":\\\\\\\"9\\\\\\\",\\\\\\\"totalElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"totalPages\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sort\\\\\\\":[{\\\\\\\"direction\\\\\\\":\\\\\\\"DESC\\\\\\\",\\\\\\\"property\\\\\\\":\\\\\\\"MODIFIED_AT\\\\\\\",\\\\\\\"ignoreCase\\\\\\\":false,\\\\\\\"nullHandling\\\\\\\":\\\\\\\"NATIVE\\\\\\\",\\\\\\\"ascending\\\\\\\":false,\\\\\\\"descending\\\\\\\":true}],\\\\\\\"content\\\\\\\":[{\\\\\\\"payment_id\\\\\\\":\\\\\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\\\\\",\\\\\\\"contract_hash\\\\\\\":\\\\\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\\\\\",\\\\\\\"payment_state\\\\\\\":\\\\\\\"COMPLETED\\\\\\\",\\\\\\\"modified_at\\\\\\\":\\\\\\\"2024-03-28T05:57:10.540Z\\\\\\\",\\\\\\\"contract\\\\\\\":{\\\\\\\"sender_end_to_end_id\\\\\\\":\\\\\\\"IpayPinc10011\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:54:22.671Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"quote\\\\\\\":{\\\\\\\"quote_id\\\\\\\":\\\\\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:46:23.907Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"type\\\\\\\":\\\\\\\"REVERSAL_AMOUNT\\\\\\\",\\\\\\\"price_guarantee\\\\\\\":\\\\\\\"FIRM\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"currency_code_filter\\\\\\\":null,\\\\\\\"service_type\\\\\\\":null,\\\\\\\"quote_elements\\\\\\\":[{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"},{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"2\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"}],\\\\\\\"liquidity_warning\\\\\\\":null,\\\\\\\"payment_method\\\\\\\":null,\\\\\\\"payment_method_fields\\\\\\\":null,\\\\\\\"payout_method_info\\\\\\\":null},\\\\\\\"fee_info\\\\\\\":null},\\\\\\\"ripplenet_info\\\\\\\":[],\\\\\\\"execution_condition\\\\\\\":\\\\\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\\\\\",\\\\\\\"crypto_transaction_id\\\\\\\":\\\\\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\\\\\",\\\\\\\"validator\\\\\\\":\\\\\\\"test.cloud.isendmock\\\\\\\",\\\\\\\"payment_type\\\\\\\":\\\\\\\"RETURN\\\\\\\",\\\\\\\"returns_payment_with_id\\\\\\\":\\\\\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\\\\\",\\\\\\\"returned_by_payment_with_id\\\\\\\":null,\\\\\\\"execution_results\\\\\\\":[{\\\\\\\"execution_result_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"execution_timestamp\\\\\\\":\\\\\\\"2024-03-28T05:56:54.173Z\\\\\\\",\\\\\\\"execution_result_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"execution_result_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"intermediary_delta\\\\\\\":null,\\\\\\\"incentive_type\\\\\\\":null,\\\\\\\"incentive_value\\\\\\\":null,\\\\\\\"transaction_hash\\\\\\\":null,\\\\\\\"venue_id\\\\\\\":null,\\\\\\\"fiat_adjusted_value\\\\\\\":null,\\\\", :exception=>#<LogStash::Json::ParserError: Unexpected character ('\' (code 92)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')
 at [Source: (byte[])"\\\"first\\\":true,\\\"last\\\":true,\\\"number\\\":\\\"0\\\",\\\"numberOfElements\\\":\\\"1\\\",\\\"size\\\":\\\"9\\\",\\\"totalElements\\\":\\\"1\\\",\\\"totalPages\\\":\\\"1\\\",\\\"sort\\\":[{\\\"direction\\\":\\\"DESC\\\",\\\"property\\\":\\\"MODIFIED_AT\\\",\\\"ignoreCase\\\":false,\\\"nullHandling\\\":\\\"NATIVE\\\",\\\"ascending\\\":false,\\\"descending\\\":true}],\\\"content\\\":[{\\\"payment_id\\\":\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\",\\\"contract_hash\\\":\\\"2552affe335b48158"[truncated 3595 bytes]; line: 1, column: 2]>}
[WARN ] 2024-04-30 11:41:37.478 [[main]>worker1] Dissector - Dissector mapping, field not found in event {"field"=>"[doc][message]", "event"=>{"host"=>"fintech", "message"=>"\\\\\\\"first\\\\\\\":true,\\\\\\\"last\\\\\\\":true,\\\\\\\"number\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"numberOfElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"size\\\\\\\":\\\\\\\"9\\\\\\\",\\\\\\\"totalElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"totalPages\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sort\\\\\\\":[{\\\\\\\"direction\\\\\\\":\\\\\\\"DESC\\\\\\\",\\\\\\\"property\\\\\\\":\\\\\\\"MODIFIED_AT\\\\\\\",\\\\\\\"ignoreCase\\\\\\\":false,\\\\\\\"nullHandling\\\\\\\":\\\\\\\"NATIVE\\\\\\\",\\\\\\\"ascending\\\\\\\":false,\\\\\\\"descending\\\\\\\":true}],\\\\\\\"content\\\\\\\":[{\\\\\\\"payment_id\\\\\\\":\\\\\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\\\\\",\\\\\\\"contract_hash\\\\\\\":\\\\\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\\\\\",\\\\\\\"payment_state\\\\\\\":\\\\\\\"COMPLETED\\\\\\\",\\\\\\\"modified_at\\\\\\\":\\\\\\\"2024-03-28T05:57:10.540Z\\\\\\\",\\\\\\\"contract\\\\\\\":{\\\\\\\"sender_end_to_end_id\\\\\\\":\\\\\\\"IpayPinc10011\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:54:22.671Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"quote\\\\\\\":{\\\\\\\"quote_id\\\\\\\":\\\\\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:46:23.907Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"type\\\\\\\":\\\\\\\"REVERSAL_AMOUNT\\\\\\\",\\\\\\\"price_guarantee\\\\\\\":\\\\\\\"FIRM\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"currency_code_filter\\\\\\\":null,\\\\\\\"service_type\\\\\\\":null,\\\\\\\"quote_elements\\\\\\\":[{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"},{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"2\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"}],\\\\\\\"liquidity_warning\\\\\\\":null,\\\\\\\"payment_method\\\\\\\":null,\\\\\\\"payment_method_fields\\\\\\\":null,\\\\\\\"payout_method_info\\\\\\\":null},\\\\\\\"fee_info\\\\\\\":null},\\\\\\\"ripplenet_info\\\\\\\":[],\\\\\\\"execution_condition\\\\\\\":\\\\\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\\\\\",\\\\\\\"crypto_transaction_id\\\\\\\":\\\\\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\\\\\",\\\\\\\"validator\\\\\\\":\\\\\\\"test.cloud.isendmock\\\\\\\",\\\\\\\"payment_type\\\\\\\":\\\\\\\"RETURN\\\\\\\",\\\\\\\"returns_payment_with_id\\\\\\\":\\\\\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\\\\\",\\\\\\\"returned_by_payment_with_id\\\\\\\":null,\\\\\\\"execution_results\\\\\\\":[{\\\\\\\"execution_result_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"execution_timestamp\\\\\\\":\\\\\\\"2024-03-28T05:56:54.173Z\\\\\\\",\\\\\\\"execution_result_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"execution_result_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"intermediary_delta\\\\\\\":null,\\\\\\\"incentive_type\\\\\\\":null,\\\\\\\"incentive_value\\\\\\\":null,\\\\\\\"transaction_hash\\\\\\\":null,\\\\\\\"venue_id\\\\\\\":null,\\\\\\\"fiat_adjusted_value\\\\\\\":null,\\\\", "@version"=>"1", "tags"=>["_jsonparsefailure"], "@timestamp"=>2024-04-30T11:41:37.214Z}}
{"host":"fintech","message":"\\\\\\\"first\\\\\\\":true,\\\\\\\"last\\\\\\\":true,\\\\\\\"number\\\\\\\":\\\\\\\"0\\\\\\\",\\\\\\\"numberOfElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"size\\\\\\\":\\\\\\\"9\\\\\\\",\\\\\\\"totalElements\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"totalPages\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sort\\\\\\\":[{\\\\\\\"direction\\\\\\\":\\\\\\\"DESC\\\\\\\",\\\\\\\"property\\\\\\\":\\\\\\\"MODIFIED_AT\\\\\\\",\\\\\\\"ignoreCase\\\\\\\":false,\\\\\\\"nullHandling\\\\\\\":\\\\\\\"NATIVE\\\\\\\",\\\\\\\"ascending\\\\\\\":false,\\\\\\\"descending\\\\\\\":true}],\\\\\\\"content\\\\\\\":[{\\\\\\\"payment_id\\\\\\\":\\\\\\\"b1e6e6c9-3327-4397-82f0-320cb1a432e1\\\\\\\",\\\\\\\"contract_hash\\\\\\\":\\\\\\\"2552affe335b4815822296f0ed85ded77928160167112ddf3f9d477e8f05c807\\\\\\\",\\\\\\\"payment_state\\\\\\\":\\\\\\\"COMPLETED\\\\\\\",\\\\\\\"modified_at\\\\\\\":\\\\\\\"2024-03-28T05:57:10.540Z\\\\\\\",\\\\\\\"contract\\\\\\\":{\\\\\\\"sender_end_to_end_id\\\\\\\":\\\\\\\"IpayPinc10011\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:54:22.671Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"quote\\\\\\\":{\\\\\\\"quote_id\\\\\\\":\\\\\\\"3ae96871-72c9-4128-a987-6d2432a8314f\\\\\\\",\\\\\\\"created_at\\\\\\\":\\\\\\\"2024-03-28T05:46:23.907Z\\\\\\\",\\\\\\\"expires_at\\\\\\\":\\\\\\\"2024-03-28T06:46:23.907Z\\\\\\\",\\\\\\\"type\\\\\\\":\\\\\\\"REVERSAL_AMOUNT\\\\\\\",\\\\\\\"price_guarantee\\\\\\\":\\\\\\\"FIRM\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"currency_code_filter\\\\\\\":null,\\\\\\\"service_type\\\\\\\":null,\\\\\\\"quote_elements\\\\\\\":[{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"},{\\\\\\\"quote_element_id\\\\\\\":\\\\\\\"55b5cabe-8d27-4945-92ac-18001b7d88e4\\\\\\\",\\\\\\\"quote_element_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"quote_element_order\\\\\\\":\\\\\\\"2\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@uat.sgp.isend\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"trans_usd_isend@uat.sgp.isend\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\"}],\\\\\\\"liquidity_warning\\\\\\\":null,\\\\\\\"payment_method\\\\\\\":null,\\\\\\\"payment_method_fields\\\\\\\":null,\\\\\\\"payout_method_info\\\\\\\":null},\\\\\\\"fee_info\\\\\\\":null},\\\\\\\"ripplenet_info\\\\\\\":[],\\\\\\\"execution_condition\\\\\\\":\\\\\\\"PrefixSha256Condition{subtypes=[ED25519-SHA-256], type=PREFIX-SHA-256, fingerprint=avpDeFbFxAx-OkzUq7dolAv8VL6ZU4SwDB2m7a1mLGc, cost=132360}\\\\\\\",\\\\\\\"crypto_transaction_id\\\\\\\":\\\\\\\"90842455-a920-4ce4-827b-4a0e7b900999\\\\\\\",\\\\\\\"validator\\\\\\\":\\\\\\\"test.cloud.isendmock\\\\\\\",\\\\\\\"payment_type\\\\\\\":\\\\\\\"RETURN\\\\\\\",\\\\\\\"returns_payment_with_id\\\\\\\":\\\\\\\"9ec07098-6b0d-498a-9300-7c50f81cebec\\\\\\\",\\\\\\\"returned_by_payment_with_id\\\\\\\":null,\\\\\\\"execution_results\\\\\\\":[{\\\\\\\"execution_result_id\\\\\\\":\\\\\\\"d94691c3-f317-4646-8a15-82fec9ea6e06\\\\\\\",\\\\\\\"execution_timestamp\\\\\\\":\\\\\\\"2024-03-28T05:56:54.173Z\\\\\\\",\\\\\\\"execution_result_type\\\\\\\":\\\\\\\"TRANSFER\\\\\\\",\\\\\\\"execution_result_order\\\\\\\":\\\\\\\"1\\\\\\\",\\\\\\\"sender_address\\\\\\\":\\\\\\\"trans_usd_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"receiver_address\\\\\\\":\\\\\\\"conct_usd_isend_isendmock@test.cloud.isendmock\\\\\\\",\\\\\\\"sending_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"receiving_amount\\\\\\\":\\\\\\\"121.000000000\\\\\\\",\\\\\\\"sending_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"receiving_fee\\\\\\\":\\\\\\\"0.000000000\\\\\\\",\\\\\\\"sending_currency_code\\\\\\\":null,\\\\\\\"receiving_currency_code\\\\\\\":null,\\\\\\\"fx_rate\\\\\\\":null,\\\\\\\"transfer_currency_code\\\\\\\":\\\\\\\"USD\\\\\\\",\\\\\\\"intermediary_delta\\\\\\\":null,\\\\\\\"incentive_type\\\\\\\":null,\\\\\\\"incentive_value\\\\\\\":null,\\\\\\\"transaction_hash\\\\\\\":null,\\\\\\\"venue_id\\\\\\\":null,\\\\\\\"fiat_adjusted_value\\\\\\\":null,\\\\","@version":"1","tags":["_jsonparsefailure"],"@timestamp":"2024-04-30T11:41:37.214Z"}

Here is the output when I run the below logstash configuration:

input {
 stdin { codec => json }
}

filter {
  json {
    source => "message"
    target => "doc"
  }

    dissect {
      mapping => {
        "[doc][message]" => "the json is:%{log}"
      }
    }

  json {
    source => "log"
  }
}
output {
 stdout{ codec => json}
}

I only copied and pasted the message: the json is: field.

Topic		Replies	Views
Parse Logstash message field into multiple field Logstash	9	2645	September 13, 2021
Parsing Json to Cloudwatch Log Group in Logstash config Logstash	7	2072	May 2, 2019
Split nested json array from log Logstash	13	3993	May 18, 2020
Logstash filter for records of nested json messages from eventhub Logstash	3	298	October 20, 2022
Parsing nested JSON in message field [Logstash][Filebeat] Logstash	1	807	April 9, 2019

Beautifying the awslamda nested logs Using Logstash in Kibana

Related topics