Hi ,
i deployed the Nodes in Ubuntu VMs and installed elastic using deb package
SPECS
root@aaa-new:/etc/elasticsearch# lscpu
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Address sizes: 52 bits physical, 57 bits virtual
Byte Order: Little Endian
CPU(s): 8
On-line CPU(s) list: 0-7
Vendor ID: AuthenticAMD
Model name: AMD EPYC 9554 64-Core Processor
CPU family: 25
Model: 17
Thread(s) per core: 1
Core(s) per socket: 4
Socket(s): 2
Stepping: 1
BogoMIPS: 6190.69
root@aaa-new:/etc/elasticsearch# free -h
total used free shared buff/cache available
Mem: 62Gi 32Gi 368Mi 1.0Mi 29Gi 29Gi
Swap: 0B 0B 0B
root@aaa-new:/etc/elasticsearch# df -h /data
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 3.4T 606G 2.8T 18% /
root@servers30-new:/etc/elasticsearch#
Storage type is SSD mounted volumes of 3.5 TB in each machine
Total data in cluster is
for now i created few indices which used for testing , apart from that major data is webcrawl data in one particular data stream. bellow i provided all indices _cat/indices
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size dataset.size
green open .ds-logs-webcrawl-prod-2024.05.08-000012 YQxLp4TxTJK5002680ErsA 5 0 10952000 0 137.8gb 137.8gb 137.8gb
green open .ds-logs-webcrawl-prod-2024.05.08-000013 GcTMqnJNQ-S_tpGJq9xBhQ 5 0 11798400 0 154.8gb 154.8gb 154.8gb
green open .ds-logs-webcrawl-prod-2024.05.08-000014 9zQiD9irQeGMec_cFho5-Q 5 0 10009600 0 130.8gb 130.8gb 130.8gb
green open .ds-logs-webcrawl-prod-2024.05.08-000015 8BSxcpg-SLifRc_yzNFi0g 5 0 10068800 0 135.6gb 135.6gb 135.6gb
green open .ds-logs-webcrawl-prod-2024.05.08-000016 NdXOHJEPRYaZHpRPPTioVQ 5 0 9800583 0 129.5gb 129.5gb 129.5gb
green open .ds-logs-webcrawl-prod-2024.05.08-000017 QCKUwCeGRLa34OuO-gc9ZA 5 0 9547200 0 128.6gb 128.6gb 128.6gb
green open .ds-logs-webcrawl-prod-2024.05.08-000018 6VLrnyYYQGa2gQCyQ71P3Q 5 0 9347200 0 124gb 124gb 124gb
green open .ds-logs-webcrawl-prod-2024.05.08-000019 8UyURUiHRmaQcMm0e7n2Gg 5 0 9377600 0 132.9gb 132.9gb 132.9gb
green open my_index FH993Jq3SQibwluX-hcmQg 1 1 3 0 35.7kb 17.8kb 17.8kb
green open .ds-logs-webcrawl-prod-2024.05.08-000020 HHi3Faq6SjCh49H1VOQY3g 5 0 9176000 0 134.6gb 134.6gb 134.6gb
green open .ds-logs-webcrawl-prod-2024.05.08-000021 vz-TUR3FTZSSRk-ccE3RYg 5 0 9662400 0 134.3gb 134.3gb 134.3gb
green open .ds-logs-webcrawl-prod-2024.05.08-000022 bKKkgvWZRReRAPeBzyuSWg 5 0 9480000 0 134.2gb 134.2gb 134.2gb
green open .ds-logs-webcrawl-prod-2024.05.08-000023 Ljbge9KbR92PrGs9f9N4Iw 5 0 9491200 0 126.4gb 126.4gb 126.4gb
green open .ds-logs-webcrawl-prod-2024.05.08-000024 kJxawPjnR8G1mKKI5HyifA 5 0 9555200 0 131.1gb 131.1gb 131.1gb
green open .ds-logs-webcrawl-prod-2024.05.08-000025 SG1p0q-tThGEZWi9xFebhQ 5 0 9653365 0 129.9gb 129.9gb 129.9gb
green open .ds-logs-webcrawl-prod-2024.05.08-000026 YuaQ92WdRRa1OcJVeed7SQ 5 0 13057600 0 164.5gb 164.5gb 164.5gb
green open .ds-logs-webcrawl-prod-2024.05.08-000027 josLhXxAReCxBAVKPBPIcg 5 0 12230846 0 160gb 160gb 160gb
green open .ds-logs-webcrawl-prod-2024.05.08-000028 K5Fu224ASGGUAdWGRKt9kw 5 0 10803200 0 142gb 142gb 142gb
green open .ds-logs-webcrawl-prod-2024.05.08-000029 a5KWyMjrRmyWl9cbluGTOw 5 0 10838400 0 140.8gb 140.8gb 140.8gb
green open .internal.alerts-observability.metrics.alerts-default-000001 9fqdXCsfQ-e1VIeS3L_7Zg 1 1 0 0 500b 250b 250b
green open .internal.alerts-security.alerts-default-000001 5JNMkNz_SIi4ML5a0BnpfQ 1 1 0 0 500b 250b 250b
green open .ds-logs-webcrawl-prod-2024.05.09-000039 tl8lPcdHSqWtIE5av0410A 5 0 10964800 0 144.6gb 144.6gb 144.6gb
green open .ds-logs-webcrawl-prod-2024.05.08-000030 EDuijk5pQXS3Yt3yXS48HA 5 0 10544000 0 142.9gb 142.9gb 142.9gb
green open .ds-logs-webcrawl-prod-2024.05.08-000031 OI5zkAopQVCd9K8IfyRycA 5 0 10808000 0 147.1gb 147.1gb 147.1gb
green open .ds-logs-webcrawl-prod-2024.05.08-000032 NsBdHgLVQrGJbqlcnnnLkg 5 0 9907200 0 135.5gb 135.5gb 135.5gb
green open .ds-logs-webcrawl-prod-2024.05.08-000033 WPNpbzUwTEaReYBJ6-ntSA 5 0 10339200 0 144.3gb 144.3gb 144.3gb
green open .ds-logs-webcrawl-prod-2024.05.08-000034 V32oRIYJTuu_MAIu4vXxeg 5 0 10843200 0 146.5gb 146.5gb 146.5gb
green open .ds-logs-webcrawl-prod-2024.05.08-000035 6ZThIdXPTgO4dj-DdojpYg 5 0 10835200 0 137gb 137gb 137gb
green open .ds-logs-webcrawl-prod-2024.05.08-000036 WX-4E8BlTsWp1xqiU3BIaA 5 0 11382400 0 149.9gb 149.9gb 149.9gb
green open .ds-logs-webcrawl-prod-2024.05.08-000037 y_5G1j2xR--5TxzyUr9tdQ 5 0 10067200 0 133.9gb 133.9gb 133.9gb
green open .ds-logs-webcrawl-prod-2024.05.08-000038 G2jk1HviRY-DIYQXCz26tA 5 0 11107200 0 149.5gb 149.5gb 149.5gb
green open .ds-logs-webcrawl-prod-2024.05.09-000040 PDHfetGGQN2L_nE_BA7MLw 5 0 10995200 0 141.9gb 141.9gb 141.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000041 sbHQzEL3SUGiBrI0AZTFag 5 0 11089600 0 146.7gb 146.7gb 146.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000042 RwK_5_dWQxiAJ8NrBfQNCw 5 0 10868800 0 139.4gb 139.4gb 139.4gb
green open .ds-webcrawl-filestream-prod-2024.05.06-000034 m3NjSxVkSNenV2G1OeqBCQ 20 0 63813405 0 519.7gb 519.7gb 519.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000044 E5Vn5crmTzSlZQkgyNhytA 5 0 10868800 0 141.9gb 141.9gb 141.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000043 JHyusHAZSBeebm6yvVS2YQ 5 0 11337600 0 141gb 141gb 141gb
green open .ds-logs-webcrawl-prod-2024.05.09-000046 Z9I02z2QTbu9wMUHR7pekA 5 0 11832000 0 150.3gb 150.3gb 150.3gb
green open .ds-logs-webcrawl-prod-2024.05.09-000045 EIyMklHNQ3-bN5Ls1hNbIw 5 0 11094400 0 149.7gb 149.7gb 149.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000048 CxCi2r5KRl6ZK9ZJJYBYug 5 0 10587936 0 145.2gb 145.2gb 145.2gb
green open .ds-logs-webcrawl-prod-2024.05.09-000049 n1X3uiqqRcWJOQLMyqptsg 5 0 9474625 0 132.5gb 132.5gb 132.5gb
green open .ds-logs-webcrawl-prod-2024.05.09-000047 zXBDFwPGTZCQpIx9TkuqAA 5 0 11676800 0 155.2gb 155.2gb 155.2gb
green open .internal.alerts-default.alerts-default-000001 iXZgzIzAR6yDJ8DqerOg1Q 1 1 0 0 500b 250b 250b
green open my-data-stream yApLa7viQqKOOyVHrw1atA 1 1 4 0 12.1kb 6kb 6kb
green open .ds-logs-webcrawl-prod-2024.05.09-000050 1uT5I-RGSWiOKjWxAozhaA 5 0 9385600 0 126.5gb 126.5gb 126.5gb
green open .ds-logs-webcrawl-prod-2024.05.09-000051 fitVHb5ySE2BihdPc4IJlQ 5 0 9678400 0 131.5gb 131.5gb 131.5gb
green open .ds-logs-webcrawl-prod-2024.05.09-000052 UPQNa3ieRZ-MzD6um7ONjg 5 0 10184932 0 125.3gb 125.3gb 125.3gb
green open .ds-logs-webcrawl-prod-2024.05.09-000053 0C4h0SyfQfKWHnqOrAnwwQ 5 0 10563200 0 130.9gb 130.9gb 130.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000054 Ky_LffszSUS-Z2oRrop2XQ 5 0 9939200 0 131.3gb 131.3gb 131.3gb
green open .internal.alerts-observability.apm.alerts-default-000001 GWaZu1sNQ9Km3_U0hJjiUA 1 1 0 0 500b 250b 250b
green open .ds-logs-webcrawl-prod-2024.05.09-000055 ExLWSQomTkaSVHxO0-IcNA 5 0 9422400 0 131.6gb 131.6gb 131.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000057 _aOgOYR4TxeGRklOSYekog 5 0 9632000 0 124.2gb 124.2gb 124.2gb
green open .ds-logs-webcrawl-prod-2024.05.09-000056 CG3HMheGTxKWXXYY9G4OUw 5 0 9217600 0 125.7gb 125.7gb 125.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000059 YoYsMC-sRO6Kv1TbFwKogg 5 0 12289600 0 164.6gb 164.6gb 164.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000058 _fN4ByBqRhuHrq-unRkkng 5 0 9504000 0 113.6gb 113.6gb 113.6gb
green open your_index ZCfZ7PjpS8-8bN_qOqKlWw 1 1 1 0 7.7kb 3.8kb 3.8kb
green open .ds-logs-webcrawl-prod-2024.05.09-000060 -6vKYNBZSNibznqTLEK-YQ 5 0 9480000 0 128gb 128gb 128gb
green open .ds-logs-webcrawl-prod-2024.05.09-000061 Px0d8K_fQ3u_BXbSK_hS5g 5 0 9158400 0 124.9gb 124.9gb 124.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000062 c3fwwkKTRkKVHIbmL1ALgA 5 0 11280186 0 151.8gb 151.8gb 151.8gb
green open .ds-logs-webcrawl-prod-2024.05.09-000063 KUTjnR9QTIO57_XzTtvzSg 5 0 11124800 0 154.7gb 154.7gb 154.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000064 Sx_rmIsmRR6CBMWK74g8RQ 5 0 9329600 0 118.8gb 118.8gb 118.8gb
green open .ds-logs-webcrawl-prod-2024.05.09-000065 zxIDSIvzTWSmx0h5qyT0RQ 5 0 13020800 0 157.2gb 157.2gb 157.2gb
green open .ds-logs-webcrawl-prod-2024.05.09-000066 yCSf5O7NSTiRKL9uzCGyhA 5 0 13516800 0 162.9gb 162.9gb 162.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000067 7knt0YjPQvOPlV1-PdICWg 5 0 12766400 0 159.9gb 159.9gb 159.9gb
green open .ds-logs-webcrawl-prod-2024.05.09-000068 LSpNG-ydSyedEX1GT1UP7w 5 0 8436800 0 112.6gb 112.6gb 112.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000069 3nC-uE2rThC9inDZ0AbNxw 5 0 8804800 0 120.5gb 120.5gb 120.5gb
green open .internal.alerts-observability.uptime.alerts-default-000001 60rQh10kTtaRnbYPRY7XZw 1 1 0 0 500b 250b 250b
green open example_index vC5yoU4tTAqXUVls2u56DQ 1 1 0 0 499b 249b 249b
green open .ds-logs-webcrawl-prod-2024.05.09-000070 90PyNGHxRjSPYVjzNtm2zw 5 0 12411200 0 164.5gb 164.5gb 164.5gb
green open .ds-logs-webcrawl-prod-2024.05.09-000071 B2CeD7FjSYO3ESInOBrK9A 5 0 12481600 0 173.7gb 173.7gb 173.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000072 LgrROjIFTdmYXB8KBo1JwQ 5 0 12022948 0 156.8gb 156.8gb 156.8gb
green open .ds-logs-webcrawl-prod-2024.05.09-000073 45dDmV35TT62cj-gKArKtQ 5 0 12115200 0 161.6gb 161.6gb 161.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000074 PeiHuzsMS5Cev7qAXg5X_Q 5 0 12542400 0 155.6gb 155.6gb 155.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000075 PVoyvPyPT3CHDLMndmtDdw 5 0 12188800 0 145.7gb 145.7gb 145.7gb
green open .ds-logs-webcrawl-prod-2024.05.09-000076 knLR-1lmTYu-CAt17jNvaA 5 0 12076800 0 147.4gb 147.4gb 147.4gb
green open .ds-logs-webcrawl-prod-2024.05.09-000077 wns4ZM9tRFyPcFoC6DdBJw 5 0 11931200 0 149.6gb 149.6gb 149.6gb
green open .ds-logs-webcrawl-prod-2024.05.09-000078 wUMe2L5CS2CYSMsiQSyvPA 5 0 11424000 0 145gb 145gb 145gb
green open idx_did_you_mean h3sqEkRbQfujcxA7Q0ao6g 1 1 0 0 500b 250b 250b
green open .ds-webcrawl-filestream-prod-2024.05.11-000036 G000R-hjRRSoC7IMYQno4w 20 0 63982400 0 504.7gb 504.7gb 504.7gb
green open .ds-webcrawl-filestream-prod-2024.05.11-000037 1yS1nYTiS-GCjZuXsddZOg 20 0 63772800 0 581.8gb 581.8gb 581.8gb
green open .ds-webcrawl-filestream-prod-2024.05.11-000038 vbaZDvQfRiaaU2_-H_aoGw 20 0 11372464 0 114.3gb 114.3gb 114.3gb
green open .ds-logs-webcrawl-prod-2024.05.10-000079 CwMfk9gKRAqj1yAQkSkBAw 5 0 12328000 0 157.9gb 157.9gb 157.9gb
green open .internal.alerts-ml.anomaly-detection.alerts-default-000001 RYwMI8yGRR629LemoslyXA 1 1 0 0 500b 250b 250b
green open .ds-webcrawl-filestream-prod-2024.04.27-000002 LxN0ilaDS-yG4uAxIGNhBQ 20 0 6728570 0 103.4gb 103.4gb 103.4gb
green open .ds-logs-webcrawl-prod-2024.05.10-000080 Q8i1YAOqTg6At-4DsbMSwg 5 0 12179200 0 155.7gb 155.7gb 155.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000081 WcpKTyN9S3uGTpZYLpl8AA 5 0 12076800 0 154.7gb 154.7gb 154.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000082 OyVlIdkMR3a5-25wc0tvFA 5 0 11916800 0 160.9gb 160.9gb 160.9gb
green open .ds-logs-webcrawl-prod-2024.05.10-000083 NracqP78QUSSXIlAS4-kmg 5 0 12337600 0 159.4gb 159.4gb 159.4gb
green open .ds-logs-webcrawl-prod-2024.05.10-000084 pqsIpckCQJGAEi0vZrYxDg 5 0 12012800 0 153.9gb 153.9gb 153.9gb
green open .ds-logs-webcrawl-prod-2024.05.10-000085 F37WjUGhQwy9URm4gzzQJw 5 0 12500800 0 159.1gb 159.1gb 159.1gb
green open .ds-logs-webcrawl-prod-2024.05.10-000086 S6QIaAGvQ6eClfxmt7kerw 5 0 11521600 0 153.1gb 153.1gb 153.1gb
green open .ds-logs-webcrawl-prod-2024.05.10-000087 _eADjX-UTMi-gCHOQH8slg 5 0 11321600 0 152gb 152gb 152gb
green open .ds-logs-webcrawl-prod-2024.05.10-000088 yA59262dSXSvDgXKQqOA8w 5 0 10695085 0 146.6gb 146.6gb 146.6gb
green open .ds-logs-webcrawl-prod-2024.05.10-000089 NcPnGUC2SA6tMNH-GrC6dA 5 0 10686259 0 147.9gb 147.9gb 147.9gb
green open .ds-logs-webcrawl-prod-2024.05.10-000100 wj-LMIzPTiu2J2aNWz6WNQ 5 0 11736000 0 144.7gb 144.7gb 144.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000101 VYL-GNzDSNW4EY-i4FwS5A 5 0 9180800 0 121.1gb 121.1gb 121.1gb
green open .ds-logs-webcrawl-prod-2024.05.10-000102 Scj-_GumS7yro5lqp62kog 5 0 11777600 0 153.8gb 153.8gb 153.8gb
green open .ds-logs-webcrawl-prod-2024.05.10-000103 jM2zxAJiQ5aE0KE-7gC_kQ 5 0 12249600 0 156.9gb 156.9gb 156.9gb
green open .ds-logs-webcrawl-prod-2024.05.10-000104 cjgTuz7CQlWTz7epDARDig 5 0 12148800 0 154.2gb 154.2gb 154.2gb
green open .ds-logs-webcrawl-prod-2024.05.10-000105 l33WDKgTScO4Nn1bX-l_lQ 5 0 9073600 0 123.3gb 123.3gb 123.3gb
green open .ds-logs-webcrawl-prod-2024.05.10-000106 8fwnHSiiSjam6fBaV6sQLQ 5 0 9333469 0 122.3gb 122.3gb 122.3gb
green open .ds-logs-webcrawl-prod-2024.05.10-000107 ZEiF0MydRMWsLDSWi4vfmQ 5 0 13715200 0 155.6gb 155.6gb 155.6gb
green open .ds-logs-webcrawl-prod-2024.05.10-000108 BKJVit_DQWa89xXErRNttQ 5 0 13213652 0 143.9gb 143.9gb 143.9gb
green open .ds-logs-webcrawl-prod-2024.05.10-000109 qiIB_ZzVT3Ka7ZJPxEL0_g 5 0 12558400 0 157.9gb 157.9gb 157.9gb
green open .internal.alerts-observability.slo.alerts-default-000001 OFJmF-51ScKfsoDxGGYrjQ 1 1 0 0 500b 250b 250b
green open .ds-logs-webcrawl-prod-2024.05.10-000090 JNhFkK40ThG3hMtyoVrnug 5 0 9997510 0 128.3gb 128.3gb 128.3gb
green open .ds-logs-webcrawl-prod-2024.05.10-000091 mMOHjAbfTi6bx9mUqnn_sw 5 0 9833600 0 126.7gb 126.7gb 126.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000092 CHNJ6EoITVmhxt9dEWppwA 5 0 12152000 0 154.1gb 154.1gb 154.1gb
green open .ds-logs-webcrawl-prod-2024.05.10-000093 Ev2s_aI0RZ-cfwafmBrr2A 5 0 8836800 0 119.7gb 119.7gb 119.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000094 cptGQIQRTZy2jXGFF6nV0g 5 0 9292800 0 122.7gb 122.7gb 122.7gb
green open .ds-logs-webcrawl-prod-2024.05.10-000095 B8aku1LPTk6es2Pg_c6kAA 5 0 9582400 0 123.8gb 123.8gb 123.8gb
green open .ds-logs-webcrawl-prod-2024.05.10-000096 uxzaVTMBSlCtAbSak4cDXA 5 0 9276800 0 123.2gb 123.2gb 123.2gb
green open .ds-logs-webcrawl-prod-2024.05.10-000097 HybIiiCJS0mHpdhEG4PIow 5 0 8843200 0 115.4gb 115.4gb 115.4gb
green open .ds-logs-webcrawl-prod-2024.05.10-000098 cqLayJDjRvmb324OqThB_w 5 0 10825600 0 137.8gb 137.8gb 137.8gb
green open .ds-logs-webcrawl-prod-2024.05.10-000099 PuXD7IUcQyCFXqX6CYtxoQ 5 0 12275200 0 158.6gb 158.6gb 158.6gb
green open .ds-logs-webcrawl-test-2024.05.07-000001 23xoiyLsQfuEIHXjLvzpCg 5 0 34199 0 644.5mb 644.5mb 644.5mb
green open .ds-webcrawl-filestream-prod-2024.04.28-000008 YrEJzfdWQwC1Vi-dhKJR2Q 20 0 6443625 0 101.7gb 101.7gb 101.7gb
green open .ds-webcrawl-filestream-prod-2024.04.28-000005 NBqLm04ESSyfCmxugkEpOA 20 0 6431706 0 99.9gb 99.9gb 99.9gb
green open .ds-webcrawl-filestream-prod-2024.04.28-000004 TM9QJ5WhRUanimp5jOIn1g 20 0 6525570 0 102.4gb 102.4gb 102.4gb
green open .ds-webcrawl-filestream-prod-2024.04.28-000007 GswwiGlxT7qe-bKnzs5s4A 20 0 6177311 0 98.8gb 98.8gb 98.8gb
green open .ds-webcrawl-filestream-prod-2024.04.28-000006 SNxAL7qOQl-fnjbUoWGIpA 20 0 6323494 0 99gb 99gb 99gb
green open .ds-logs-webcrawl-prod-2024.05.10-000113 o9Fq--nsS_25WsexbZbCiA 5 0 9038400 0 118.3gb 118.3gb 118.3gb
green open .ds-logs-webcrawl-prod-2024.05.10-000114 EEaBS4hPTx6R_lErSspS7w 5 0 9016000 0 128gb 128gb 128gb
red open .ds-logs-webcrawl-prod-2024.05.10-000115 lthhC0FoR9ioFRCYr53OWQ 5 0 6890335 0 85.1gb 85.1gb 85.1gb
green open .ds-logs-webcrawl-prod-2024.05.10-000112 -SjGnIqlS-yPlY-McdxWWg 5 0 9118400 0 121gb 121gb 121gb
green open .ds-logs-webcrawl-prod-2024.05.10-000111 MCt2uDzeSym1m6qPt2VGJg 5 0 8958400 0 117.2gb 117.2gb 117.2gb
green open .ds-logs-webcrawl-prod-2024.05.10-000110 PQ6DhJYeTqK_0iqabQ2E7g 5 0 12969600 0 175gb 175gb 175gb
green open .internal.alerts-stack.alerts-default-000001 6useGXHdQnKW4zxY4u4vWQ 1 1 0 0 500b 250b 250b
green open .kibana-observability-ai-assistant-conversations-000001 nMxDy_EoRC-xfDAJ051rXw 1 1 0 0 500b 250b 250b
green open .ds-logs-videos-test-2024.05.07-000001 PsyFC2TXSge_ZdAupqtS4w 5 0 12032605 0 6.1gb 6.1gb 6.1gb
green open .ds-logs-webcrawl-prod-2024.05.07-000001 x5c8namfQC-eexaM_DnGbA 5 0 10633600 0 135.7gb 135.7gb 135.7gb
green open .ds-logs-webcrawl-prod-2024.05.07-000002 psTMEHubTPiiQsDXfDTy1Q 5 0 9284800 0 120.7gb 120.7gb 120.7gb
green open .ds-logs-webcrawl-prod-2024.05.07-000003 DO_EnXwVR4CsYZdVINicFg 5 0 9270400 0 118.5gb 118.5gb 118.5gb
green open .ds-logs-webcrawl-prod-2024.05.07-000004 QnGmIGg5TgapfBWAfGQ6Zw 5 0 10192000 0 139gb 139gb 139gb
green open .ds-logs-webcrawl-prod-2024.05.07-000005 t2lOc3HWRAKtWuMehoepAQ 5 0 8950036 0 118.6gb 118.6gb 118.6gb
green open .ds-logs-webcrawl-prod-2024.05.07-000006 lIwkP2ylQgiNd3nGd61fbQ 5 0 9771200 0 125.3gb 125.3gb 125.3gb
green open .internal.alerts-transform.health.alerts-default-000001 4bgZ9f83ROi9Nmhyg7LRJA 1 1 0 0 500b 250b 250b
green open .internal.alerts-ml.anomaly-detection-health.alerts-default-000001 NOvD4WcpQry1K6VXc2ZJUg 1 1 0 0 500b 250b 250b
green open .ds-logs-videos-prod-2024.05.07-000001 49FnwjyHQVi1MVt4pAg5zA 5 0 15018015 0 41.3gb 41.3gb 41.3gb
green open .ds-logs-webcrawl-prod-2024.05.11-000116 3kAcI8gATZaLeZf7DNzQmQ 5 0 10985680 0 140.1gb 140.1gb 140.1gb
green open .ds-logs-webcrawl-prod-2024.05.11-000117 -d_QHUyTQvOHbHPlcQPdgQ 5 0 11885699 0 142.7gb 142.7gb 142.7gb
green open .ds-logs-webcrawl-prod-2024.05.11-000118 lH50X98hQmStGWL7tDiXmQ 5 0 12968000 0 156.9gb 156.9gb 156.9gb
green open .ds-logs-webcrawl-prod-2024.05.11-000119 -BAdLGYtRcCLQcuhhsqHOw 5 0 11664000 0 141.2gb 141.2gb 141.2gb
green open .ds-logs-videos-prod-2024.05.07-000002 d1uhfmiwRLisO3XC0qD8Aw 5 0 3026545 0 7.9gb 7.9gb 7.9gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000011 AxFlSx4jSoWZlWbb7ljjqw 20 0 6279706 0 96.2gb 96.2gb 96.2gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000012 2S0t8qLKQx6n8sZYedTBlg 20 0 1445482 0 22.5gb 22.5gb 22.5gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000013 W4qUivEfTxa_GcqR-P_kMA 20 0 6318045 0 92gb 92gb 92gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000014 cP0RlpD3Rn2D7xqaNHxRdg 20 0 6547355 0 96.9gb 96.9gb 96.9gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000015 EhZQwbklT_mki0asId9CVA 20 0 6582742 0 97.2gb 97.2gb 97.2gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000016 kbs1zw1lTzWp4X4yreDH2Q 20 0 6416248 0 96.1gb 96.1gb 96.1gb
green open .ds-webcrawl-filestream-prod-2024.04.29-000017 p_7EY5D7QRSMS4bW6ium1g 20 0 6314201 0 94.6gb 94.6gb 94.6gb
green open .internal.alerts-observability.logs.alerts-default-000001 9hGOwCgVSjmaIeY7rRnfqA 1 1 0 0 500b 250b 250b
green open .internal.alerts-observability.threshold.alerts-default-000001 STIkNMO0SRuOLpTnnK4cZw 1 1 0 0 500b 250b 250b
green open .kibana-observability-ai-assistant-kb-000001 exLCF-EjQcSWYXiMM1RjHw 1 1 0 0 500b 250b 250b
green open .ds-webcrawl-filestream-prod-2024.05.01-000023 cVNizrvPR8Wfx6HJ_5iJaw 20 0 6297258 0 95.3gb 95.3gb 95.3gb
green open .ds-webcrawl-filestream-prod-2024.04.30-000019 MXgdBiT3Q6iR8L7ZVcu_3Q 20 0 6226191 0 92gb 92gb 92gb
green open .ds-webcrawl-filestream-prod-2024.05.01-000025 jsfdGm8aRpi7tV2R0BG28g 20 0 1011067 2838029 33.7gb 33.7gb 33.7gb
green open .ds-webcrawl-filestream-prod-2024.05.01-000026 uDvSabQNReqp3-GbHioX-Q 20 0 0 0 5.4kb 5.4kb 5.4kb
green open .ds-logs-webcrawl-prod-2024.05.11-000121 XtnF58lqTj61oW9PPRzy1A 5 0 10217600 0 134.9gb 134.9gb 134.9gb
green open .ds-logs-webcrawl-prod-2024.05.08-000007 t14n7GoEStiwwBFg1I8LqA 5 0 9769534 0 122.3gb 122.3gb 122.3gb
green open .ds-logs-webcrawl-prod-2024.05.08-000008 -gakTZuqSwaCbO-bSnGU4g 5 0 9670400 0 118.1gb 118.1gb 118.1gb
green open .ds-logs-webcrawl-prod-2024.05.08-000009 3CkBX7jyRRycnYBlnO4NSA 5 0 10761600 0 140.5gb 140.5gb 140.5gb
green open .ds-logs-webcrawl-prod-2024.05.11-000122 8JgV8-Z3Rcm4HBBvv-2JUg 5 0 12832000 0 173.9gb 173.9gb 173.9gb
green open .ds-logs-webcrawl-prod-2024.05.11-000123 qYzpakobQfK6_MnV99312Q 5 0 9811200 0 133.7gb 133.7gb 133.7gb
green open .ds-logs-webcrawl-prod-2024.05.11-000124 9KqM4gsFSpa3znOX8CpKxA 5 0 11067200 0 150.9gb 150.9gb 150.9gb
green open .ds-logs-webcrawl-prod-2024.05.11-000125 LpOAcquzTVeA_i7kCK0RHA 5 0 10308800 0 143.9gb 143.9gb 143.9gb
green open .ds-logs-webcrawl-prod-2024.05.11-000126 lzgq3OvxRqqMSrbaUQOx7A 5 0 10678400 0 148.4gb 148.4gb 148.4gb
green open .ds-logs-webcrawl-prod-2024.05.11-000127 BEu4bh3UR72YUR7EWE_FrA 5 0 7495070 0 141.9gb 141.9gb 141.9gb
green open .ds-logs-webcrawl-prod-2024.05.11-000120 oCkKd8RFSrSSc-Onxk3nlg 5 0 10649600 0 137.2gb 137.2gb 137.2gb
green open .ds-twitter-filestream-test-2024.04.26-000001 Y_7a2UGVTiyZSBU_po1VFw 1 1 0 0 500b 250b 250b
green open .ds-webcrawl-filestream-prod-2024.04.30-000020 78raC9egRTWl9I5rqe-MDw 20 0 6364440 0 94.4gb 94.4gb 94.4gb
green open .ds-webcrawl-filestream-prod-2024.04.30-000021 EMuuaCxHQxePerZW8Inztw 20 0 6414297 0 96.1gb 96.1gb 96.1gb
green open .ds-webcrawl-filestream-prod-2024.04.30-000022 H0dJMb2aSbaQB8ohenj-PQ 20 0 6304382 0 95gb 95gb 95gb
green open .ds-logs-webcrawl-prod-2024.05.08-000010 QTnjel69SLmB5Rb9rVoY_Q 5 0 11707200 0 151.2gb 151.2gb 151.2gb
green open .ds-logs-webcrawl-prod-2024.05.08-000011 -q5Av7HDQ9SXeBBjIZi1Og 5 0 11043200 0 138gb 138gb 138gb
also providing the template i used for the webcrawl index which is main concern for me
{
"priority": 1500,
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "webcrawl-fs-policy"
},
"mapping": {
"nested_fields": {
"limit": "500"
},
"depth": {
"limit": "50"
},
"field_name_length": {
"limit": "1000"
},
"total_fields": {
"limit": "20000"
}
},
"refresh_interval": "60s",
"number_of_shards": "5",
"max_docvalue_fields_search": "500",
"default_pipeline": "webcrawl-pipeline",
"analysis": {
"filter": {
"my_shingle_filter": {
"max_shingle_size": "2",
"min_shingle_size": "2",
"type": "shingle"
},
"my_standard_filter": {
"type": "stop"
}
},
"analyzer": {
"my_shingle_analyzer": {
"filter": [
"lowercase",
"my_shingle_filter",
"trim",
"stemmer"
],
"tokenizer": "whitespace"
},
"my_standard_analyzer": {
"filter": [
"lowercase",
"my_standard_filter",
"trim",
"stemmer"
],
"tokenizer": "whitespace"
}
}
},
"number_of_replicas": "0"
}
},
"mappings": {
"_routing": {
"required": false
},
"numeric_detection": false,
"dynamic_date_formats": [
"strict_date_optional_time",
"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"
],
"_source": {
"excludes": [],
"includes": [],
"enabled": true
},
"dynamic": true,
"dynamic_templates": [],
"date_detection": true,
"properties": {
"Envelope.Payload-Metadata.HTTP-Response-Metadata.Headers": {
"type": "flattened"
},
"cn": {
"eager_global_ordinals": false,
"index_phrases": false,
"search_quote_analyzer": "my_shingle_analyzer",
"fielddata_frequency_filter": {
"min": 0.01,
"max": 1,
"min_segment_size": 50
},
"fielddata": true,
"norms": true,
"analyzer": "my_shingle_analyzer",
"index": true,
"store": false,
"type": "text",
"index_options": "positions"
},
"ht": {
"eager_global_ordinals": false,
"index_phrases": false,
"search_quote_analyzer": "my_standard_analyzer",
"fielddata": false,
"norms": true,
"analyzer": "my_standard_analyzer",
"index": true,
"store": false,
"type": "text",
"index_options": "positions"
},
"wd": {
"index": true,
"ignore_malformed": false,
"store": false,
"type": "date",
"doc_values": true
},
"wt": {
"eager_global_ordinals": false,
"index_phrases": false,
"search_quote_analyzer": "my_standard_analyzer",
"fielddata": false,
"norms": true,
"analyzer": "my_standard_analyzer",
"index": true,
"store": false,
"type": "text",
"index_options": "positions"
},
"wu": {
"dynamic": true,
"type": "object",
"enabled": true,
"properties": {
"original": {
"eager_global_ordinals": false,
"index_phrases": false,
"search_quote_analyzer": "my_standard_analyzer",
"fielddata": false,
"norms": true,
"analyzer": "my_standard_analyzer",
"index": true,
"store": false,
"type": "text",
"index_options": "positions"
},
"domain": {
"eager_global_ordinals": false,
"index_phrases": false,
"search_quote_analyzer": "my_standard_analyzer",
"fielddata": false,
"norms": true,
"analyzer": "my_standard_analyzer",
"index": true,
"store": false,
"type": "text",
"index_options": "positions"
}
}
}
}
}
},
"index_patterns": [
"logs-webcrawl-*",
"logs-webcrawl-prod*"
],
"data_stream": {
"hidden": false,
"allow_custom_routing": false
},
"composed_of": [],
"allow_auto_create": false
}