Error 500 Internal Server Error

Elastic Stack Kibana
1

I just install graph to an es server (2.3.1) with shield. But when I tried to pass a query I got an error 500.
In the logs i found the following:
Kibana.stdout:
{"type":"error","@timestamp":"2016-04-06T10:55:06+00:00","tags":[],"pid":16028,"level":"error","message":"[illegal_state_exception]
Field data loading is forbidden on
[message]","error":{"message":"[illegal_state_exception]
Field data loading is forbidden on
[message]","name":"Error","stack":"[illegal_state_exception] Field data
loading is forbidden on [message] ::
{"path":"/logstash-2016.04.06/_graph/explore","query":{},"body":"{\"query\":{\"query_string\":{\"default_field\":\"_all\",\"query\":\"VLAN100\"}},\"controls\":{\"use_significance\":true,\"sample_size\":2000,\"timeout\":5000},\"connections\":{\"vertices\":[{\"field\":\"message\",\"size\":5,\"min_doc_count\":3}]},\"vertices\":[{\"field\":\"message\",\"size\":5,\"min_doc_count\":3}]}","statusCode":500,"response":"{\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"Field

data loading is forbidden on
[message]\"}],\"type\":\"search_phase_execution_exception\",\"reason\":\"all
shards
failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":[{\"shard\":0,\"index\":\"logstash-2016.04.06\",\"node\":\"uzq_HT8uRBeK3PpkKMd-Hw\",\"reason\":{\"type\":\"illegal_state_exception\",\"reason\":\"Field

data loading is forbidden on
[message]\"}}]},\"status\":500}"}\n at respond
(/apps/Oss/ELK/kibana-4.5.0-linux-x64/node_modules/elasticsearch/src/lib/transport.js:238:15)\n
at checkRespForFailure
(/apps/Oss/ELK/kibana-4.5.0-linux-x64/node_modules/elasticsearch/src/lib/transport.js:201:7)\n

at HttpConnector.
(/apps/Oss/ELK/kibana-4.5.0-linux-x64/node_modules/elasticsearch/src/lib/connectors/http.js:155:7)\n
at IncomingMessage.wrapper
(/apps/Oss/ELK/kibana-4.5.0-linux-x64/node_modules/lodash/index.js:3095:19)\n
at emitNone (events.js:72:20)\n
at IncomingMessage.emit (events.js:166:7)\n at endReadableNT
(_stream_readable.js:905:12)\n at nextTickCallbackWith2Args
(node.js:437:9)\n at process._tickDomainCallback
(node.js:392:17)"},"url":{"protocol":null,"slashes":null,"auth":null,"host":null,"port":null,"hostname":null,"hash":null,"search":null,"query":{},"pathname":"/api/graph/graphExplore","path":"/api/graph/graphExplore","href":"/api/graph/graphExplore"}}
{"type":"response","@timestamp":"2016-04-06T10:55:06+00:00","tags":[],"pid":16028,"method":"post","statusCode":500,"req":{"url":"/api/graph/graphExplore","method":"post","headers":{"host":"lqzc4007.mod.nl:5601","user-agent":"Mozilla/5.0

(Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101
Firefox/39.0","accept":"application/json, text/plain,
/","accept-language":"nl,en-US;q=0.7,en;q=0.3","accept-encoding":"gzip,

deflate","content-type":"application/json;charset=utf-8","kbn-version":"4.5.0","referer":"https://lqzc4007.mod.nl:5601/app/graph","content-length":"316","cookie":"sid=Fe26.2264f6170d6f10154c5e44c3982f4dad3894496d8e122ec0e2b7e27e583f76d4cvNIRzIQvDaGUsJE5RmNpPgBxTIuaiEltoTphjp9n56l0ghUQjxs23_nI9rHj-xEUCb4_4iUHMMf-rvhKDWmiUsxE_5M8R0RNfc17AyqAs0_zDuP1OLiRXc1rtvUzxB31wc6f285b22913325034fd41140911373e7954a38922a7faf97998d4f8bff6f13d*6U8auPE4EzdrdytQ3YlIGtFYRAIYS_dnc13_iCk_Bgg","connection":"keep-alive","pragma":"no-cache","cache-control":"no-cache"},"remoteAddress":"10.50.101.106","userAgent":"10.50.101.106","referer":"https://lqzc4007.mod.nl:5601/app/graph"},"res":{"statusCode":500,"responseTime":59,"contentLength":9},"message":"POST

/api/graph/graphExplore 500 59ms - 9.0B"}

Please help

Kind regards

Theo

2

And elasticsearch.log
[2016-04-06 12:55:06,849][WARN
][rest.suppressed ] /logstash-2016.04.06/_graph/explore Params:
{index=logstash-2016.04.06}
Failed to execute phase [query],
all shards failed; shardFailures
{[uzq_HT8uRBeK3PpkKMd-Hw][logstash-2016.04.06][0]:
RemoteTransportException[[EServer][10.50.101.158:9300][indices:data/read/search[phase/query]]];
nested:
SearchParseException[failed to parse search source
[{"size":0,"timeout":5000,"query":{"bool":{"must":{"query_string":{"default_field":"_all","query":"VLAN100"}}}},"aggregations":{"sample":{"sampler":{"shard_size":2000},"aggregations":{"field0":{"significant_terms":{"field":"message","size":5,"min_doc_count":3,"shard_min_doc_count":2,"execution_hint":"map"}}}}}}]];

nested: IllegalStateException[Field data loading is forbidden on
[message]]; }{[uzq_HT8uRBeK3PpkKMd-Hw][logstash-2016.04.06][1]:
RemoteTransportException[[EServer][10.50.101.158:9300][indices:data/read/search[phase/query]]];
nested: SearchParseException[failed
to parse search source
[{"size":0,"timeout":5000,"query":{"bool":{"must":{"query_string":{"default_field":"_all","query":"VLAN100"}}}},"aggregations":{"sample":{"sampler":{"shard_size":2000},"aggregations":{"field0":{"significant_terms":{"field":"message","size":5,"min_doc_count":3,"shard_min_doc_count":2,"execution_hint":"map"}}}}}}]];

nested: IllegalStateException[Field data loading is forbidden on
[message]]; }{[uzq_HT8uRBeK3PpkKMd-Hw][logstash-2016.04.06][2]:
RemoteTransportException[[EServer][10.50.101.158:9300][indices:data/read/search[phase/query]]];
nested: SearchParseException[failed
to parse search source
[{"size":0,"timeout":5000,"query":{"bool":{"must":{"query_string":{"default_field":"_all","query":"VLAN100"}}}},"aggregations":{"sample":{"sampler":{"shard_size":2000},"aggregations":{"field0":{"significant_terms":{"field":"message","size":5,"min_doc_count":3,"shard_min_doc_count":2,"execution_hint":"map"}}}}}}]];

nested: IllegalStateException[Field data loading is forbidden on
[message]]; }{[uzq_HT8uRBeK3PpkKMd-Hw][logstash-2016.04.06][3]:
RemoteTransportException[[EServer][10.50.101.158:9300][indices:data/read/search[phase/query]]];
nested: SearchParseException[failed
to parse search source
[{"size":0,"timeout":5000,"query":{"bool":{"must":{"query_string":{"default_field":"_all","query":"VLAN100"}}}},"aggregations":{"sample":{"sampler":{"shard_size":2000},"aggregations":{"field0":{"significant_terms":{"field":"message","size":5,"min_doc_count":3,"shard_min_doc_count":2,"execution_hint":"map"}}}}}}]];

nested: IllegalStateException[Field data loading is forbidden on
[message]]; }{[uzq_HT8uRBeK3PpkKMd-Hw][logstash-2016.04.06][4]:
RemoteTransportException[[EServer][10.50.101.158:9300][indices:data/read/search[phase/query]]];
nested: SearchParseException[failed
to parse search source
[{"size":0,"timeout":5000,"query":{"bool":{"must":{"query_string":{"default_field":"_all","query":"VLAN100"}}}},"aggregations":{"sample":{"sampler":{"shard_size":2000},"aggregations":{"field0":{"significant_terms":{"field":"message","size":5,"min_doc_count":3,"shard_min_doc_count":2,"execution_hint":"map"}}}}}}]];

nested: IllegalStateException[Field data loading is forbidden on
[message]]; }

3

If I recall correctly it is possible for an administrator to specifically exclude some fields from ever having field data loaded.

This looks like what would happen if that was done for the "message" field.

4

I just installed the Graph plugin on a Windows installation and also got the same "500 Internal Server Error" when trying to do a simple search.
The error specifies that Document mapping type can't start with a '_' , probably in a path, but seems like the underscore in the path is created by the Graph plugin installation. Here's the Kibana error output:

error [09:38:24.187] [invalid_type_name_exception] Document mapping type name
can't start with '_' :: {"path":"/logstash-2016.03.04/graph/explore","query":{
},"body":"{"query":{"query_string":{"default_field":"all","query":"s
earch"}},"controls":{"use_significance":true,"sample_size":2000,"timeout
":5000,"sample_diversity":{"field":"clientname.raw","max_docs_per_value
":1}},"connections":{"vertices":[{"field":"operation","size":5,"min_d
oc_count":3}]},"vertices":[{"field":"operation","size":5,"min_doc_coun
t":3}]}","statusCode":400,"response":"{"error":{"root_cause":[{"type":"i
nvalid_type_name_exception","reason":"Document mapping type name can't start
with ''"}],"type":"invalid_type_name_exception","reason":"Document map
ping type name can't start with ''"},"status":400}"}
at respond (c:\ELK\Kibana\kibana-4.5.0-windows\node_modules\elasticsearch\sr
c\lib\transport.js:238:15)
at checkRespForFailure (c:\ELK\Kibana\kibana-4.5.0-windows\node_modules\elas
ticsearch\src\lib\transport.js:201:7)
at HttpConnector. (c:\ELK\Kibana\kibana-4.5.0-windows\node_module
s\elasticsearch\src\lib\connectors\http.js:155:7)
at IncomingMessage.wrapper (c:\ELK\Kibana\kibana-4.5.0-windows\node_modules
lodash\index.js:3095:19)
at emitNone (events.js:72:20)
at IncomingMessage.emit (events.js:166:7)
at endReadableNT (_stream_readable.js:905:12)
at nextTickCallbackWith2Args (node.js:437:9)
at process._tickDomainCallback (node.js:392:17)

Anybody run into a similar situation?

5

Hi Pham, it looks like your issue is different than the others in this thread. This indicates that the Graph plugin isn't installed into Elasticsearch.

Can you ensure that you have installed the Graph and Licensing plugins into each Elasticsearch node, and that you have restarted the cluster?

6

Thanks for the quick response. I found out that I didn't restart elastic search after the plugin installation. Restarted it and looks like it's working now.
Thanks.

7

Hi,

I can't remember that I restarted the Elasticsearch Server. And I cannot remember this was mentioned in any training or manual.

But I will do so next thing in the morning.

Thanks

Kind regards

Theo

8

Mark,

I cannot remeber i did such thing. The installation is pretty default. How can I determine this?

Kind regards,

Theo

9

I'm confused as to which of your issues you are referring (the failure to load "message" field data or the missing graph plugin?).
I assume you mean the original field-data loading issue.
To test field-data loading issues see if you can run this simple aggregation on the field - that will eliminate the graph plugin from the problem:

curl -XPOST "http://localhost:9200/logstash-2016.04.06/_search" -d'
{
   "size": 0,
   "aggs": {
	  "top_terms": {
		 "terms": {
			"field": "message"
		 }
	  }
   }
}'
10

Hi, I wanna install timelion plugin for kibana. I copied tilelion tar file to instaledPlugins folder and extracted it there. After running kibana I've gotten the error so I deleted it from there but Kibana is still not working. I have this error:

    '../src/ui/public/styles/theme.less\nModule build failed: Error: Cannot

find module 'color-name'\n at Function.Module._resolveFilename (module.js:3
25:15)\n at Function.Module._load (module.js:276:25)\n at Module.require (
module.js:353:17)\n at require (internal/module.js:12:17)\n at Object. (C:\Users\AJ31490\Downloads\kibana-5.0.0-alpha3-windows\kibana-5.0.
0-alpha3-windows\node_modules\css-loader\node_modules\cssnano\node_modules
\postcss-colormin\node_modules\colormin\node_modules\color\node_modules\co
lor-string\color-string.js:2:18)\n at Module._compile (module.js:409:26)\n
at Module._extensions..js (module.js:416:10)\n at Object.require.extensions
.(anonymous function) [as .js] (C:\Users\AJ31490\Downloads\kibana-5.0.0-alph
a3-windows\kibana-5.0.0-alpha3-windows\node_modules\babel-core\lib\api\reg
ister\node.js:214:7)\n at Module.load (module.js:343:32)\n at Function.Mo
dule._load (module.js:300:12)\n at Module.require (module.js:353:17)\n at
require (internal/module.js:12:17)\n at Object. (C:\Users\AJ3149
0\Downloads\kibana-5.0.0-alpha3-windows\kibana-5.0.0-alpha3-windows\node_mod
ules\css-loader\node_modules\cssnano\node_modules\postcss-colormin\node_mo
dules\colormin\node_modules\color\index.js:3:14)\n at Module._compile (mo
dule.js:409:26)\n at Module._extensions..js (module.js:416:10)\n at Object
.require.extensions.(anonymous function) [as .js] (C:\Users\AJ31490\Downloads
\kibana-5.0.0-alpha3-windows\kibana-5.0.0-alpha3-windows\node_modules\babel-
core\lib\api\register\node.js:214:7)\n @ ../src/ui/public/styles nonrecursiv
e \/\\[^\/\\]+\.less',
'../src/ui/public/styles/truncate.less\nModule build failed: Error: Cann
ot find module 'color-name'\n at Function.Module._resolveFilename (module.j
s:325:15)\n at Function.Module._load (module.js:276:25)\n at Module.requir
e (module.js:353:17)\n at require (internal/module.js:12:17)\n at Object.<
anonymous> (C:\Users\AJ31490\Downloads\kibana-5.0.0-alpha3-windows\kibana-5
.0.0-alpha3-windows\node_modules\css-loader\node_modules\cssnano\node_modul
es\postcss-colormin\node_modules\colormin\node_modules\color\node_modules
\color-string\color-string.js:2:18)\n at Module._compile (module.js:409:26)
n at Module._extensions..js (module.js:416:10)\n at Object.require.extensi
ons.(anonymous function) [as .js] (C:\Users\AJ31490\Downloads\kibana-5.0.0-a
lpha3-windows\kibana-5.0.0-alpha3-windows\node_modules\babel-core\lib\api\
register\node.js:214:7)\n at Module.load (module.js:343:32)\n at Function
.Module.load (module.js:300:12)\n at Module.require (module.js:353:17)\n
at require (internal/module.js:12:17)\n at Object. (C:\Users\AJ3
1490\Downloads\kibana-5.0.0-alpha3-windows\kibana-5.0.0-alpha3-windows\node
modules\css-loader\node_modules\cssnano\node_modules\postcss-colormin\node
_modules\colormin\node_modules\color\index.js:3:14)\n at Module._compile
(module.js:409:26)\n at Module._extensions..js (module.js:416:10)\n at Obj
ect.require.extensions.(anonymous function) [as .js] (C:\Users\AJ31490\Downlo
ads\kibana-5.0.0-alpha3-windows\kibana-5.0.0-alpha3-windows\node_modules\bab
el-core\lib\api\register\node.js:214:7)\n @ ../src/ui/public/styles nonrecur
sive \/\\[^\/\\]+\.less' ],
publicPath: '/bundles/' },
isBoom: true,
isServer: true,
output:
{ statusCode: 500,
payload:
{ statusCode: 500,
error: 'Internal Server Error',
message: 'An internal server error occurred' },
headers: {} },
reformat: [Function] }