"failed shard on node... ...Data too large, data for [<transport_request>] would be" only for 3 most recent .monitoring-es indices

Thank you @DavidTurner for the quick reply. I just discovered that command as you were responding and came back to post my findings.

It turns out that many of my shards are culprits. Looking something like this:

    "env-stg-2020.02.20" : {
      "uuid" : "TgkrNGtKTwGNDeSshJ2YLQ",
      "primaries" : {
        "fielddata" : {
          "memory_size_in_bytes" : 64716808,
          "evictions" : 0,
          "fields" : {
            "logmessage.keyword" : {
              "memory_size_in_bytes" : 225728
            },
            "fields.service.keyword" : {
              "memory_size_in_bytes" : 2096
            },
            "fields.env.keyword" : {
              "memory_size_in_bytes" : 1088
            },
            "_id" : {
              "memory_size_in_bytes" : 64487896
            }
          }
        }
      },

My template for this index looks like this:

  "default" : {
    "order" : -1,
    "index_patterns" : [
      "env-stg*",
      "env-prd*",
      "env2-stg*",
      "env2-prd*"
    ],
    "settings" : {
      "index" : {
        "number_of_shards" : "2",
        "number_of_replicas" : "0"
      }
    },
    "mappings" : {
      "doc" : {
        "properties" : {
          "logTimestamphttp" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "auth" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "ident" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "source" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "lc_identifier" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "logTimestamp" : {
            "type" : "date"
          },
          "clientip" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "@version" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "beat" : {
            "properties" : {
              "hostname" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "name" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "version" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              }
            }
          },
          "host" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "logTimestampString" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "class" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "offset" : {
            "type" : "long"
          },
          "logmessage" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "verb" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "prospector" : {
            "properties" : {
              "type" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              }
            }
          },
          "thread" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "tags" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "input" : {
            "properties" : {
              "type" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              }
            }
          },
          "prelogmessage" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "@timestamp" : {
            "type" : "date"
          },
          "bytes" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "response" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "loglevel" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "httpversion" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "ignore_above" : 256,
                "type" : "keyword"
              }
            }
          },
          "fields" : {
            "properties" : {
              "product" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "service" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "env" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              },
              "customer" : {
                "type" : "text",
                "fields" : {
                  "keyword" : {
                    "ignore_above" : 256,
                    "type" : "keyword"
                  }
                }
              }
            }
          }
        }
      }
    },
    "aliases" : { }
  },

It looks like I overlooked the template in testing when comparing for deltas between the production and testing environments. However, it appears everything is working how I'd expect in testing. I believe the template currently in use in production was cherry-picked and persisted since we were on version 5. I don't believe we were using it to do anything specific other than set the number of shards/replicas originally.

As a reference, the testing environment looks like this:

{
  "_shards" : {
    "total" : 239,
    "successful" : 239,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "fielddata" : {
        "memory_size_in_bytes" : 0,
        "evictions" : 0,
        "fields" : {
          "type" : {
            "memory_size_in_bytes" : 0
          },
          "_id" : {
            "memory_size_in_bytes" : 0
          }
        }
      }
    },

I imagine removing this template altogether is an option. Or, replacing with the template that exists in our testing environment currently. Is there a better option?