More
[2020-03-26T15:01:32,007][INFO ][l.o.elasticsearch ] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[vector-diags-000005][2] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[vector-diags-000005][2]] containing [index {[vector-diags][_doc][PgVbF3EBumyRkgKT6q5y], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-19601-03.scranton.pa.pitt.comcast.net\"},\"@timestamp\":\"2020-03-26T15:00:25.033Z\",\"msg\":\"ERROR: [77309161] Slow Request: client_ip: 127.0.0.1:50906 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod3/noggin.com/XPMV0000000001254263/1578673485/noggin.comXPMV0000000001254263.ts status: 206 unique id: redirection_tries: 0 bytes: 8044 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.000 dns_lookup_begin: 0.000 dns_lookup_end: 14.270 server_connect: 14.270 server_connect_end: -0.001 server_first_read: 14.290 server_read_header_done: 14.290 server_close: 14.800 ua_write: 14.290 ua_close: 14.291 sm_finish: 14.800 plugin_active: 0.000 plugin_total: 0.000\",\"@version\":\"1\",\"hostname\":\"ccdn-ats-tk-19601-03\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Mar 26 15:00:25.033\",\"server\":\"{0x2ab1aca68700}\",\"site\":\"19601\"}]}]]"})
[2020-03-26T15:16:26,903][INFO ][l.o.elasticsearch ] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[vector-diags-000005][2] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[vector-diags-000005][2]] containing [index {[vector-diags][_doc][GBNpF3EBumyRkgKTkteO], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-19601-01.scranton.pa.pitt.comcast.net\"},\"@timestamp\":\"2020-03-26T15:15:24.977Z\",\"msg\":\"ERROR: [89208964] Slow Request: client_ip: 127.0.0.1:60544 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod3/cinesony.com/SDSD1186800000003203/1582246765/cinesony.comSDSD1186800000003203.ts status: 206 unique id: redirection_tries: 0 bytes: 8051 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.000 dns_lookup_begin: 0.000 dns_lookup_end: 14.098 server_connect: 14.098 server_connect_end: -0.001 server_first_read: 14.118 server_read_header_done: 14.118 server_close: 14.120 ua_write: 14.118 ua_close: 14.118 sm_finish: 14.120 plugin_active: 0.000 plugin_total: 0.000\",\"@version\":\"1\",\"hostname\":\"ccdn-ats-tk-19601-01\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Mar 26 15:15:24.977\",\"server\":\"{0x2aeb16a52700}\",\"site\":\"19601\"}]}]]"})
[2020-03-26T15:16:29,904][INFO ][l.o.elasticsearch ] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[vector-diags-000005][2] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[vector-diags-000005][2]] containing [index {[vector-diags][_doc][fBNpF3EBumyRkgKTnuRq], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-16702-04.pittsburgh.pa.pitt.comcast.net\"},\"@version\":\"1\",\"msg\":\"ERROR: [6696913] Slow Request: client_ip: 127.0.0.1:58333 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod7/indemand.com/INMV1219231000702808/1576735457/indemand.comINMV1219231000702808.ts status: 206 unique id: redirection_tries: 0 bytes: 8158 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.000 dns_lookup_begin: 0.000 dns_lookup_end: 10.092 server_connect: 10.092 server_connect_end: -0.001 server_first_read: 10.144 server_read_header_done: 10.144 server_close: 10.189 ua_write: 10.144 ua_close: 10.144 sm_finish: 10.189 plugin_active: 0.000 plugin_total: 0.000\",\"@timestamp\":\"2020-03-26T15:15:24.589Z\",\"hostname\":\"ccdn-ats-tk-16702-04\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Mar 26 15:15:24.589\",\"server\":\"{0x2aac5fd63700}\",\"site\":\"16702\"}]}]]"})
[2020-03-26T15:17:28,905][INFO ][l.o.elasticsearch ] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[vector-diags-000005][2] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[vector-diags-000005][2]] containing [index {[vector-diags][_doc][9RRqF3EBumyRkgKThMBn], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-19601-01.scranton.pa.pitt.comcast.net\"},\"@timestamp\":\"2020-03-26T15:15:24.977Z\",\"msg\":\"ERROR: [89208964] Slow Request: client_ip: 127.0.0.1:60544 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod3/cinesony.com/SDSD1186800000003203/1582246765/cinesony.comSDSD1186800000003203.ts status: 206 unique id: redirection_tries: 0 bytes: 8051 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.000 dns_lookup_begin: 0.000 dns_lookup_end: 14.098 server_connect: 14.098 server_connect_end: -0.001 server_first_read: 14.118 server_read_header_done: 14.118 server_close: 14.120 ua_write: 14.118 ua_close: 14.118 sm_finish: 14.120 plugin_active: 0.000 plugin_total: 0.000\",\"@version\":\"1\",\"hostname\":\"ccdn-ats-tk-19601-01\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Mar 26 15:15:24.977\",\"server\":\"{0x2aeb16a52700}\",\"site\":\"19601\"}]}]]"})
[2020-03-31T00:33:48,766][INFO ][l.o.elasticsearch ] retrying failed action with response code: 429 ({"type"=>"es_rejected_execution_exception", "reason"=>"rejected execution of processing of [335551529][indices:data/write/bulk[s][p]]: request: BulkShardRequest [[vector-diags-000005][3]] containing [index {[vector-diags][_doc][E08CLnEBumyRkgKTNB3c], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-22001-02.49thst.pa.panjde.comcast.net\"},\"@version\":\"1\",\"msg\":\"ERROR: [2650964] Slow Request: client_ip: 127.0.0.1:60789 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod15/indemand.com/INMV0217231000333232/1486410627/indemand.comINMV0217231000333232.index status: 206 unique id: redirection_tries: 0 bytes: 8057 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.902 dns_lookup_begin: 31.062 dns_lookup_end: 31.062 server_connect: 31.062 server_connect_end: 0.936 server_first_read: 31.072 server_read_header_done: 31.072 server_close: -0.001 ua_write: 31.072 ua_close: 31.073 sm_finish: 31.073 plugin_active: 0.000 plugin_total: 0.000\",\"@timestamp\":\"2020-03-31T00:33:46.876Z\",\"hostname\":\"ccdn-ats-tk-22001-02\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Mar 31 00:33:46.876\",\"server\":\"{0x2afbe6847700}\",\"site\":\"22001\"}]}], target allocation id: TpaPEPUoT0iJXRK53CEHmQ, primary term: 174 on EsThreadPoolExecutor[name = es-data-r2-4/write, queue capacity = 300, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@3ecd851[Running, pool size = 6, active threads = 6, queued tasks = 300, completed tasks = 165686243]]"})
[2020-04-05T19:02:38,158][INFO ][l.o.elasticsearch ] retrying failed action with response code: 429 ({"type"=>"es_rejected_execution_exception", "reason"=>"rejected execution of processing of [562643113][indices:data/write/bulk[s][p]]: request: BulkShardRequest [[vector-diags-000005][1]] containing [index {[vector-diags][_doc][_A65S3EB0_tjy993KLWH], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-11101-02.exeter.nh.boston.comcast.net\"},\"@timestamp\":\"2020-04-05T19:02:35.921Z\",\"msg\":\"ERROR: [130866440] Slow Request: client_ip: 127.0.0.1:60876 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod19/sho.com/XPMV0001546340358000/1548491102/sho.comXPMV0001546340358000.8x_trick status: 206 unique id: redirection_tries: 0 bytes: 8053 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.882 dns_lookup_begin: 32.835 dns_lookup_end: 32.835 server_connect: 32.835 server_connect_end: 32.852 server_first_read: 32.870 server_read_header_done: 32.870 server_close: -0.001 ua_write: 32.870 ua_close: 32.870 sm_finish: 32.870 plugin_active: 0.000 plugin_total: 0.000\",\"@version\":\"1\",\"hostname\":\"ccdn-ats-tk-11101-02\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Apr 5 19:02:35.921\",\"server\":\"{0x2b7b4ae3a700}\",\"site\":\"11101\"}]}], target allocation id: 8dWPjYEZQemVtW7D0F5KOA, primary term: 57 on EsThreadPoolExecutor[name = es-data-r1-4/write, queue capacity = 300, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@613b73f4[Running, pool size = 6, active threads = 6, queued tasks = 300, completed tasks = 328410521]]"})
[2020-04-08T00:25:30,899][INFO ][l.o.elasticsearch ] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[vector-diags-000005][3] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[vector-diags-000005][3]] containing [index {[vector-diags][_doc][jJcsV3EBwrzmcNz-kss5], source[{\"beat\":{\"version\":\"6.5.4\",\"name\":\"ccdn-ats-tk-21441-02.pompanobeach.fl.pompano.comcast.net\"},\"@version\":\"1\",\"msg\":\"ERROR: [104826098] Slow Request: client_ip: 127.0.0.1:54514 protocol: http url: http://mpeg2origin.sys.comcast.net/mp2vod8/sho.com/XPMV0001574940828000/1578448737/sho.comXPMV0001574940828000.-8x_trick status: 206 unique id: redirection_tries: 0 bytes: 40811 fd: -1 client state: 5 server state: 0 ua_begin: 0.000 ua_first_read: 0.000 ua_read_header_done: 0.000 cache_open_read_begin: 0.000 cache_open_read_end: 0.894 dns_lookup_begin: 31.950 dns_lookup_end: 31.950 server_connect: 31.950 server_connect_end: 32.010 server_first_read: 32.077 server_read_header_done: 32.077 server_close: -0.001 ua_write: 32.077 ua_close: 32.077 sm_finish: 32.077 plugin_active: 0.000 plugin_total: 0.000\",\"@timestamp\":\"2020-04-07T20:37:36.251Z\",\"hostname\":\"ccdn-ats-tk-21441-02\",\"log_type\":\"vector-diags\",\"source\":\"/opt/trafficserver/var/log/trafficserver/diags.log\",\"timestamp\":\"Apr 7 20:37:36.251\",\"server\":\"{0x2b9d6a03a700}\",\"site\":\"21441\"}]}]]"})