There is a problem to attached files.
index schema:
*
*
state: open
settings: {
index.analysis.filter.customngramfilter.type: nGram
index.analysis.filter.customngramfilter.max_gram: 150
index.analysis.analyzer.stringngramanalyzer.name: StringNGramAnalyzer
index.analysis.filter.customngramfilter.name: customngramfilter
index.analysis.analyzer.stringngramanalyzer.type: custom
index.analysis.analyzer.stringngramanalyzer.tokenizer: keyword
index.analysis.filter.customngramfilter.min_gram: 2
index.analysis.analyzer.sortlowercaseanalyzer.filter: lowercase
index.analysis.analyzer.sortlowercaseanalyzer.type: custom
index.analysis.analyzer.sortlowercaseanalyzer.tokenizer: keyword
index.number_of_replicas: 1
index.number_of_shards: 5
index.analysis.analyzer.sortlowercaseanalyzer.name: SortLowerCaseAnalyzer
index.analysis.analyzer.stringngramanalyzer.filter:
lowercase,customngramfilter
index.version.created: 200599
}
mappings: {
ad: {
_source: {
compress: true
}
properties: {
placementRotation: {
index: no
type: integer
}
placementCreationDate: {
format: dateOptionalTime
type: date
}
thumbnailFileWidth: {
index: no
type: integer
}
classification1: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
classification1: {
analyzer: stringngramanalyzer
type: string
}
}
}
viewCopiesLink: {
index: no
type: string
}
classification2: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
classification2: {
analyzer: stringngramanalyzer
type: string
}
}
}
relatedCopies: {
type: integer
}
classification3: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
classification3: {
analyzer: stringngramanalyzer
type: string
}
}
}
categoryID: {
type: integer
}
classification4: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
classification4: {
analyzer: stringngramanalyzer
type: string
}
}
}
videoStartMethod: {
type: integer
}
classification5: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
classification5: {
analyzer: stringngramanalyzer
type: string
}
}
}
deliveryGroupsIDs: {
type: integer
}
thumbnailAccountFileType: {
index: no
type: integer
}
agencyID: {
type: integer
}
advertiserName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
advertiserName: {
analyzer: stringngramanalyzer
type: string
}
}
}
thumbnailContentFileName: {
index: no
type: string
}
formatName: {
index: not_analyzed
omit_norms: true
index_options: docs
type: string
}
isCrossChannelAd: {
index: not_analyzed
type: boolean
}
placementLastImpressionTime: {
format: dateOptionalTime
type: date
}
isArchivedForMedia: {
index: not_analyzed
type: boolean
}
formatID: {
type: integer
}
placementName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
placementName: {
analyzer: stringngramanalyzer
type: string
}
}
}
placementType: {
index: no
type: integer
}
advertiserID: {
type: integer
}
dataCaptureID: {
type: integer
}
masterAdID: {
type: integer
}
classificationsAccountID: {
type: integer
}
syncUnitID: {
type: integer
}
creativeShop: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
creativeShop: {
analyzer: stringngramanalyzer
type: string
}
}
}
status: {
type: integer
}
isUsingDeliveryGroups: {
index: not_analyzed
type: boolean
}
isRejected: {
index: no
type: boolean
}
thumbnailFileHeight: {
index: no
type: integer
}
placementActualStartDate: {
index: no
format: dateOptionalTime
type: date
}
iD: {
type: integer
}
isInStreamCompanionAd: {
index: not_analyzed
type: boolean
}
size: {
type: integer
}
syncCampaignID: {
type: integer
}
adPart: {
index: no
type: string
}
adName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
adName: {
analyzer: stringngramanalyzer
type: string
}
}
}
publisherID: {
type: integer
}
campaignName: {
type: multi_field
fields: {
campaignName: {
analyzer: stringngramanalyzer
type: string
}
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
}
}
isCopyChanged: {
index: no
type: boolean
}
sectionName: {
index: no
type: string
}
placementStartDate: {
index: no
format: dateOptionalTime
type: date
}
placementID: {
type: integer
}
creatorID: {
type: integer
}
placementEndDate: {
index: no
format: dateOptionalTime
type: date
}
modeName: {
index: no
type: string
}
originalTemplateID: {
index: no
type: integer
}
newAdStatus: {
type: integer
}
syncPublisherID: {
type: integer
}
mode: {
type: integer
}
sectionID: {
type: integer
}
syncSectionID: {
type: integer
}
templateID: {
type: integer
}
advancedFeatures: {
type: integer
}
smartVersioningTypeID: {
type: integer
}
archived: {
type: boolean
}
publisherName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
publisherName: {
analyzer: stringngramanalyzer
type: string
}
}
}
dimensions: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
dimensions: {
type: string
}
}
}
sVGroupName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
sVGroupName: {
analyzer: stringngramanalyzer
type: string
}
}
}
isArchivedForAgency: {
index: not_analyzed
type: boolean
}
placementLastClickTime: {
format: dateOptionalTime
type: date
}
thumbnailContentPathID: {
index: no
type: integer
}
campaignID: {
type: integer
}
customAdFormatID: {
type: integer
}
actualImpressionsTotal: {
index: no
type: long
}
createdByAdEditor: {
index: not_analyzed
type: boolean
}
creativeTagging: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
creativeTagging: {
analyzer: stringngramanalyzer
type: string
}
}
}
categoryAccountID: {
type: integer
}
targetAudienceIDs: {
type: integer
}
screenGrabStatusName: {
index: not_analyzed
omit_norms: true
index_options: docs
type: string
}
agencyName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
agencyName: {
analyzer: stringngramanalyzer
type: string
}
}
}
syncUnitName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
syncUnitName: {
analyzer: stringngramanalyzer
type: string
}
}
}
newAdStatusText: {
index: no
type: string
}
targetAudienceNames: {
type: multi_field
fields: {
targetAudienceNames: {
index: not_analyzed
omit_norms: true
index_options: docs
type: string
}
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
}
}
screenGrabStatus: {
index: no
type: integer
}
creationDate: {
format: dateOptionalTime
type: date
}
accountsRolesBitmap: {
properties: {
accountID: {
type: integer
}
role: {
type: long
}
}
}
creatorName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
creatorName: {
analyzer: stringngramanalyzer
type: string
}
}
}
dataCaptureFormName: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
dataCaptureFormName: {
analyzer: stringngramanalyzer
type: string
}
}
}
guidelineID: {
type: integer
}
maxDurationInSec: {
type: integer
}
deliveryGroupsNames: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
deliveryGroupsNames: {
index: not_analyzed
omit_norms: true
index_options: docs
type: string
}
}
}
lastUpdatedBy: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
lastUpdatedBy: {
analyzer: stringngramanalyzer
type: string
}
}
}
thumbnailAccountID: {
index: no
type: integer
}
lastUpdatedDate: {
format: dateOptionalTime
type: date
}
notes: {
type: multi_field
fields: {
sort: {
include_in_all: false
analyzer: sortlowercaseanalyzer
type: string
}
notes: {
index: no
type: string
}
}
}
}
}
}
aliases: [ ]
}
LOG:
*
*
[2013-04-09 10:56:50,870][INFO ][org.elasticsearch.gateway] [ElserNJ01]
recovered [3] indices into cluster_state
[2013-04-09 10:56:53,091][INFO ][org.elasticsearch.cluster.service]
[ElserNJ01] added
{[ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]],},
reason: zen-disco-receive(join from
node[[ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]]])
[2013-04-09 11:04:02,372][INFO ][org.elasticsearch.cluster.metadata]
[ElserNJ01] [ad] deleting index
[2013-04-09 11:04:16,775][INFO ][org.elasticsearch.cluster.metadata]
[ElserNJ01] [ad] creating index, cause [api], shards [5]/[1], mappings [ad]
[2013-04-09 11:04:17,068][INFO ][org.elasticsearch.cluster.metadata]
[ElserNJ01] [ad] update_mapping [ad]
[2013-04-09 11:05:27,319][INFO ][org.elasticsearch.cluster.metadata]
[ElserNJ01] [ad] update_mapping [ad] (dynamic)
[2013-04-09 19:36:24,640][WARN ][org.elasticsearch.index.merge.scheduler]
[ElserNJ01] [ad][1] failed to merge
java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.Arrays.copyOfRange(Unknown Source)
at java.lang.String.(Unknown Source)
at org.apache.lucene.index.TermBuffer.toTerm(TermBuffer.java:122)
at org.apache.lucene.index.SegmentTermEnum.term(SegmentTermEnum.java:184)
at org.apache.lucene.index.SegmentMergeInfo.next(SegmentMergeInfo.java:73)
at
org.apache.lucene.index.SegmentMerger.mergeTermInfos(SegmentMerger.java:501)
at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:428)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:108)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4263)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3908)
at
org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:388)
at
org.apache.lucene.index.TrackingConcurrentMergeScheduler.doMerge(TrackingConcurrentMergeScheduler.java:91)
at
org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:456)
[2013-04-09 19:36:24,746][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][1] failed engine
java.lang.IllegalStateException: this writer hit an OutOfMemoryError;
cannot flush
at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3563)
at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
at
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
at
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
at
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
at org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
at
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
at
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
at
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
at
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:769)
at
org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:403)
at
org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:733)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
[2013-04-09 19:36:24,742][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][0] failed engine
java.lang.OutOfMemoryError: GC overhead limit exceeded
at
org.apache.lucene.index.FreqProxTermsWriterPerField$FreqProxPostingsArray.(FreqProxTermsWriterPerField.java:194)
at
org.apache.lucene.index.FreqProxTermsWriterPerField$FreqProxPostingsArray.newInstance(FreqProxTermsWriterPerField.java:204)
at
org.apache.lucene.index.ParallelPostingsArray.grow(ParallelPostingsArray.java:48)
at
org.apache.lucene.index.TermsHashPerField.growParallelPostingsArray(TermsHashPerField.java:157)
at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:460)
at
org.apache.lucene.index.DocInverterPerField.processFields(DocInverterPerField.java:189)
at
org.apache.lucene.index.DocFieldProcessorPerThread.processDocument(DocFieldProcessorPerThread.java:278)
at
org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:766)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:2060)
at
org.elasticsearch.index.engine.robin.RobinEngine.innerIndex(RobinEngine.java:577)
at
org.elasticsearch.index.engine.robin.RobinEngine.index(RobinEngine.java:489)
at
org.elasticsearch.index.shard.service.InternalIndexShard.index(InternalIndexShard.java:330)
at
org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:159)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction.performOnPrimary
(TransportShardReplicationOperationAction.java:532)
at
org.elasticsearch.action.support.replication.TransportShardReplicationOperationAction$AsyncShardOperationAction$1.run
(TransportShardReplicationOperationAction.java:430)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
[2013-04-09 19:36:29,091][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][1],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [Failed to perform [bulk/shard] on replica, message
[RemoteTransportException[[ElserNJ01][inet[/10.24.4.194:9300]]
[bulk/shard/replica]]; nested: OutOfMemoryError[GC overhead limit
exceeded]; ]]
[2013-04-09 19:36:29,761][WARN ][org.elasticsearch.index.merge.scheduler]
[ElserNJ01] [ad][4] failed to merge
java.lang.OutOfMemoryError: GC overhead limit exceeded
at org.apache.lucene.index.TermBuffer.toTerm(TermBuffer.java:122)
at org.apache.lucene.index.SegmentTermEnum.term(SegmentTermEnum.java:184)
at org.apache.lucene.index.SegmentMergeInfo.next(SegmentMergeInfo.java:73)
at
org.apache.lucene.index.SegmentMerger.mergeTermInfos(SegmentMerger.java:501)
at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:428)
at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:108)
at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4263)
at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3908)
at
org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:388)
at
org.apache.lucene.index.TrackingConcurrentMergeScheduler.doMerge(TrackingConcurrentMergeScheduler.java:91)
at
org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:456)
[2013-04-09 19:36:30,791][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][4] failed engine
java.lang.IllegalStateException: this writer hit an OutOfMemoryError;
cannot flush
at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3563)
at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3552)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:450)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:399)
at
org.apache.lucene.index.DirectoryReader.doOpenFromWriter(DirectoryReader.java:413)
at
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:432)
at
org.apache.lucene.index.DirectoryReader.doOpenIfChanged(DirectoryReader.java:375)
at org.apache.lucene.index.IndexReader.openIfChanged(IndexReader.java:508)
at
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:109)
at
org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:57)
at
org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:137)
at
org.elasticsearch.index.engine.robin.RobinEngine.refresh(RobinEngine.java:769)
at
org.elasticsearch.index.shard.service.InternalIndexShard.refresh(InternalIndexShard.java:403)
at
org.elasticsearch.index.shard.service.InternalIndexShard$EngineRefresher$1.run(InternalIndexShard.java:733)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
[2013-04-09 19:36:32,053][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][1], node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message
[IllegalStateException[this writer hit an OutOfMemoryError; cannot flush]]]
[2013-04-09 19:36:32,053][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][1],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message
[IllegalStateException[this writer hit an OutOfMemoryError; cannot flush]]]
[2013-04-09 19:36:32,184][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [engine failure, message
[IllegalStateException[this writer hit an OutOfMemoryError; cannot flush]]]
[2013-04-09 19:36:32,185][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [engine failure, message
[IllegalStateException[this writer hit an OutOfMemoryError; cannot flush]]]
[2013-04-09 19:36:32,185][WARN ][org.elasticsearch.indices.cluster]
[ElserNJ01] [ad][4] master [[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet
[/10.24.4.194:9300]]] marked shard as started, but shard have not been
created, mark shard as failed
[2013-04-09 19:36:32,185][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [master
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet[/10.24.4.194:9300]] marked shard
as started, but shard have not been created,
mark shard as failed]
[2013-04-09 19:36:32,185][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [master
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet[/10.24.4.194:9300]] marked shard
as started, but shard have not been created,
mark shard as failed]
[2013-04-09 19:36:32,192][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][0], node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [engine failure, message [OutOfMemoryError[GC
overhead limit exceeded]]]
[2013-04-09 19:36:32,192][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][0],
node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [engine failure, message [OutOfMemoryError[GC
overhead limit exceeded]]]
[2013-04-09 19:36:32,221][WARN ][org.elasticsearch.indices.cluster]
[ElserNJ01] [ad][0] master [[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet
[/10.24.4.194:9300]]] marked shard as started, but shard have not been
created, mark shard as failed
[2013-04-09 19:36:32,222][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][0], node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [master
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet[/10.24.4.194:9300]] marked shard
as started, but shard have not been created,
mark shard as failed]
[2013-04-09 19:36:32,222][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][0],
node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[STARTED], reason [master
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet[/10.24.4.194:9300]] marked shard
as started, but shard have not been created,
mark shard as failed]
[2013-04-09 20:33:23,201][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][4] failed engine
java.lang.OutOfMemoryError: Java heap space
[2013-04-09 20:33:26,208][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message [OutOfMemoryError[Java
heap space]]]
[2013-04-09 20:33:26,208][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message [OutOfMemoryError[Java
heap space]]]
[2013-04-09 20:48:53,458][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][0],
node[JBSkSWCRTTyVV6hloDSAWQ],
[P], s[STARTED], reason [engine failure, message [OutOfMemoryError[Java
heap space]]]
[2013-04-09 21:28:41,858][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][4] failed engine
java.lang.OutOfMemoryError: Java heap space
[2013-04-09 21:28:42,950][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message [OutOfMemoryError[Java
heap space]]]
[2013-04-09 21:28:42,950][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[STARTED], reason [engine failure, message [OutOfMemoryError[Java
heap space]]]
[2013-04-09 21:32:15,818][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][0],
node[JBSkSWCRTTyVV6hloDSAWQ],
[R], s[INITIALIZING], reason [Failed to start shard, message
[RecoveryFailedException[[ad][0]: Recovery failed from
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw]
[inet[/10.24.4.194:9300]] into
[ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]]]; nested:
RemoteTransportException[[ElserNJ01][inet
[/10.24.4.194:9300]][index/shard/recovery/startRecovery]]; nested:
RecoveryEngineException[[ad][0] Phase[2] Execution failed]; nested:
RemoteTransportException[[ElserNJ02][inet[/10.24.4.195:9300]][index/shard/recovery/prepareTranslog]];
nested: OutOfMemoryError[GC overhead limit
exceeded]; ]]
[2013-04-09 21:32:16,099][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][2],
node[JBSkSWCRTTyVV6hloDSAWQ],
[P], s[STARTED], reason [engine failure, message [OutOfMemoryError[GC
overhead limit exceeded]]]
[2013-04-09 21:32:16,099][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][2],
node[JBSkSWCRTTyVV6hloDSAWQ],
[P], s[STARTED], reason [master
[ElserNJ01][1Vvv1tVsRdWodp6gJGKksw][inet[/10.24.4.194:9300]] marked shard
as started, but shard have not been created,
mark shard as failed]
[2013-04-09 21:40:56,946][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][4] failed engine
java.lang.OutOfMemoryError: Java heap space
[2013-04-09 21:40:59,892][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[INITIALIZING], reason [engine failure, message
[OutOfMemoryError[Java heap space]]]
[2013-04-09 21:40:59,892][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[INITIALIZING], reason [engine failure, message
[OutOfMemoryError[Java heap space]]]
[2013-04-09 21:49:21,964][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][1],
node[JBSkSWCRTTyVV6hloDSAWQ],
[P], s[STARTED], reason [engine failure, message
[IllegalStateException[this writer hit an OutOfMemoryError; cannot flush]]]
[2013-04-09 21:50:13,980][WARN ][org.elasticsearch.index.engine.robin]
[ElserNJ01] [ad][4] failed engine
java.lang.OutOfMemoryError: Java heap space
[2013-04-09 21:50:20,677][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[INITIALIZING], reason [engine failure, message
[OutOfMemoryError[Java heap space]]]
[2013-04-09 21:50:20,677][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[R], s[INITIALIZING], reason [engine failure, message
[OutOfMemoryError[Java heap space]]]
[2013-04-09 22:00:47,973][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][0],
node[JBSkSWCRTTyVV6hloDSAWQ],
[R], s[STARTED], reason [engine failure, message [OutOfMemoryError[GC
overhead limit exceeded]]]
[2013-04-09 22:02:58,382][WARN ][org.elasticsearch.transport] [ElserNJ01]
Received response for a request that has timed out, sent [44843ms] ago,
timed
out [14804ms] ago, action [discovery/zen/fd/ping], node
[[ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]]], id [438628]
[2013-04-09 22:04:37,752][INFO ][org.elasticsearch.cluster.service]
[ElserNJ01] removed
{[ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]],},
reason:
zen-disco-node_failed([ElserNJ02][JBSkSWCRTTyVV6hloDSAWQ][inet[/10.24.4.195:9300]]),
reason failed to ping, tried [3] times, each with maximum
[30s] timeout
[2013-04-09 22:04:37,846][WARN ][org.elasticsearch.indices.cluster]
[ElserNJ01] [ad][4] failed to start shard
org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException: [ad][4]
failed recovery
at
org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:228)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Caused by: org.elasticsearch.index.engine.EngineCreationFailureException:
[ad][4] failed to create engine
at
org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:252)
at
org.elasticsearch.index.shard.service.InternalIndexShard.start(InternalIndexShard.java:279)
at
org.elasticsearch.index.gateway.local.LocalIndexShardGateway.recover(LocalIndexShardGateway.java:182)
at
org.elasticsearch.index.gateway.IndexShardGatewayService$1.run(IndexShardGatewayService.java:177)
... 3 more
Caused by: java.io.FileNotFoundException: _dem.fnm
at
org.elasticsearch.index.store.Store$StoreDirectory.openInput(Store.java:519)
at org.apache.lucene.index.FieldInfos.(FieldInfos.java:71)
at org.apache.lucene.index.IndexWriter.getFieldInfos(IndexWriter.java:1212)
at
org.apache.lucene.index.IndexWriter.getCurrentFieldInfos(IndexWriter.java:1228)
at org.apache.lucene.index.IndexWriter.(IndexWriter.java:1161)
at org.apache.lucene.index.XIndexWriter.(XIndexWriter.java:17)
at
org.elasticsearch.index.engine.robin.RobinEngine.createWriter(RobinEngine.java:1365)
at
org.elasticsearch.index.engine.robin.RobinEngine.start(RobinEngine.java:250)
... 6 more
[2013-04-09 22:04:37,846][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] sending failed shard for [ad][4], node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[INITIALIZING], reason [Failed to start shard, message
[IndexShardGatewayRecoveryException[[ad][4] failed recovery]; nested:
EngineCreationFailureException[[ad][4] failed to create engine]; nested:
FileNotFoundException[_dem.fnm]; ]]
[2013-04-09 22:04:37,846][WARN ][org.elasticsearch.cluster.action.shard]
[ElserNJ01] received shard failed for [ad][4],
node[1Vvv1tVsRdWodp6gJGKksw],
[P], s[INITIALIZING], reason [Failed to start shard, message
[IndexShardGatewayRecoveryException[[ad][4] failed recovery]; nested:
EngineCreationFailureException[[ad][4] failed to create engine]; nested:
FileNotFoundException[_dem.fnm]; ]]
Thanks.
--
You received this message because you are subscribed to the Google Groups "elasticsearch" group.
To unsubscribe from this group and stop receiving emails from it, send an email to elasticsearch+unsubscribe@googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.