Ingest data with GCP Dataflow

Elastic Stack Elasticsearch
7

Found them!

Here is one of the documents, there are around 5 million similar ones:

{
  "_index": ".ds-logs-gcp.pubsub-metrics-2022.01.11-000001",
  "_type": "_doc",
  "_id": "eGtVSX4BVV1vns0ot6nA",
  "_version": 1,
  "_score": 1,
  "_source": {
    "points": {
      "udp_active": true,
      "udp_high": 6,
      "udp_medium": 6,
      "udp_low": 6
    },
    "labels": {
      "device_id": "5000009d6bbf9cee",
      "place_id": 7800
    },
    "@timestamp": "2022-01-11T13:29:03Z",
    "agent": {
      "type": "dataflow",
      "name": "",
      "version": "1.0.0",
      "id": ""
    },
    "data_stream": {
      "type": "logs",
      "dataset": "gcp.pubsub",
      "namespace": "metrics"
    },
    "ecs": {
      "version": "1.10.0"
    },
    "message": "{\"points\":{\"udp_active\":true,\"udp_high\":6,\"udp_medium\":6,\"udp_low\":6},\"labels\":{\"device_id\":\"5000009d6bbf9cee\",\"place_id\":7800},\"timestamp\":\"2022-01-11T13:29:03Z\"}",
    "service": {
      "type": "gcp.pubsub"
    },
    "event": {
      "module": "gcp",
      "dataset": "gcp.pubsub"
    }
  },
  "fields": {
    "points.udp_medium": [
      6
    ],
    "points.udp_active": [
      true
    ],
    "points.udp_high": [
      6
    ],
    "points.udp_low": [
      6
    ],
    "data_stream.namespace": [
      "metrics"
    ],
    "labels.place_id": [
      7800
    ],
    "message": [
      "{\"points\":{\"udp_active\":true,\"udp_high\":6,\"udp_medium\":6,\"udp_low\":6},\"labels\":{\"device_id\":\"5000009d6bbf9cee\",\"place_id\":7800},\"timestamp\":\"2022-01-11T13:29:03Z\"}"
    ],
    "data_stream.type": [
      "logs"
    ],
    "service.type": [
      "gcp.pubsub"
    ],
    "agent.type": [
      "dataflow"
    ],
    "labels.device_id": [
      "5000009d6bbf9cee"
    ],
    "@timestamp": [
      "2022-01-11T13:29:03.000Z"
    ],
    "event.module": [
      "gcp"
    ],
    "agent.id": [
      ""
    ],
    "ecs.version": [
      "1.10.0"
    ],
    "data_stream.dataset": [
      "gcp.pubsub"
    ],
    "agent.name": [
      ""
    ],
    "agent.version": [
      "1.0.0"
    ],
    "event.dataset": [
      "gcp.pubsub"
    ]
  }
}

The one below it has the same index but different ids.

Edit: Actually, all 5 million documents have the same timestamp.