Convert timestamp formats in array of objects in a Nested Field

Elastic Stack Elasticsearch
1

Hi There,

I have an index that looks like so:

{
  "_index": "ec2_logs-index-2020.05.11",
  "_type": "_doc",
  "_id": "V3LAAnIBk2krYRHTuxCC",
  "_version": 1,
  "_score": null,
  "_source": {
    "owner": "104115521938",
    "@version": "1",
    "logStream": "mystream",
    "subscriptionFilters": [
      "To-Firehose"
    ],
    "messageType": "DATA_MESSAGE",
    "logGroup": "ec2-fluentd",
    "@timestamp": "2020-05-11T08:01:24.400Z",
    "logEvents": [
      {
        "timestamp": 1589167148000,
        "id": "35439611645423604170418303021688828486859711444132757504",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"dhclient\",\"pid\":\"2987\",\"message\":\"XMT: Solicit on eth0, interval 109630ms.\"}"
      },
      {
        "timestamp": 1589167161000,
        "id": "35439611935333291751316403861653166031288411021878165505",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"sshd\",\"pid\":\"3864\",\"message\":\"Accepted publickey for ec2-user from 138.75.33.59 port 51868 ssh2: RSA SHA256:QY/2uJJiV5cYpErAb/KLg/505B6WQ4ZlvcBazh1Qfyo\"}"
      },
      {
        "timestamp": 1589167172000,
        "id": "35439611935333291751316403861653166031288411021878165506",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Created slice User Slice of ec2-user.\"}"
      },
      {
        "timestamp": 1589167201000,
        "id": "35439611935333291751316403861653166031288411021878165507",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Starting User Slice of ec2-user.\"}"
      },
      {
        "timestamp": 1589167254000,
        "id": "35439611935333291751316403861653166031288411021878165508",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd\",\"message\":\"Started Session 2 of user ec2-user.\"}"
      },
      {
        "timestamp": 1589167553000,
        "id": "35439611935333291751316403861653166031288411021878165509",
        "message": "{\"host\":\"ip-22-22-2-83\",\"ident\":\"systemd-logind\",\"message\":\"New session 2 of user ec2-user.\"}"
      }
    ],
    "type": "ec2_logs"
  },
  "fields": {
    "@timestamp": [
      "2020-05-11T08:01:24.400Z"
    ]
  },
  "sort": [
    1589184084400
  ]
}

Index mapping is

{
  "ec2_logs-index-2020.05.11" : {
    "mappings" : {
      "dynamic" : "true",
      "_meta" : { },
      "_source" : {
        "includes" : [ ],
        "excludes" : [ ]
      },
      "dynamic_date_formats" : [
        "strict_date_optional_time",
        "yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
      ],
      "dynamic_templates" : [ ],
      "date_detection" : true,
      "numeric_detection" : false,
      "properties" : {
        "@timestamp" : {
          "type" : "date",
          "format" : "strict_date_optional_time"
        },
        "@version" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "logEvents" : {
          "type" : "nested",
          "properties" : {
            "id" : {
              "type" : "text"
            },
            "message" : {
              "type" : "text"
            },
            "timestamp" : {
              "type" : "date_nanos"
            }
          }
        },
        "logGroup" : {
          "type" : "text"
        },
        "logStream" : {
          "type" : "text"
        },
        "messageType" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "owner" : {
          "type" : "text",
          "fields" : {
            "keyword" : {
              "type" : "keyword",
              "ignore_above" : 256
            }
          }
        },
        "subscriptionFilters" : {
          "type" : "text"
        },
        "type" : {
          "type" : "text"
        }
      }
    }
  }
}

What I'm trying to achieve here is convert the timestamp fields in logEvents nested field from epoch to dd/MM/YYYY hh:mm:ss format. How do I go about achieving this in all array objects of a nested field?

Thanks in advance...
ck

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.