I have a dockerized web application that contains 3 containers (Django/ PostgreSQL/Elasticsearch), The indices of the Elasticsearch are indexed depending on the Django database models using a library called Django-Elasticsearch-dsl. Everything is working good on my localhost and indexing of Django database to the Elasticsearch container is working without any errors. When I deployed my application on an AWS EC2 instance It works well for the first few times and then an error occurs after indexing more data (BulkIndexerror). I have changed my instance type and reached t2.2xlarge (I have started with t2 small and gradually reached t2.2x large) and when I change the instance type it works well for a period of time and after time the same error occurs
in my localhost when I run python manage.py search_index --rebuild this appears to me:
Are you sure you want to delete the 'books, entries' indices? [y/N]: y
Deleting index 'books'
Deleting index 'entries'
Creating index 'books'
Creating index 'entries'
Indexing 6 'book' objects
Indexing 39 'entry' objects
on AWS EC2 instance:
ubuntu@ip-172-31-29-148:~/searchablebook$ sudo docker exec -it 307b30109be8 python manage.py search_index --rebuild
System check identified some issues:
Are you sure you want to delete the 'books, entries' indices? [y/N]: y
Deleting index 'books'
Deleting index 'entries'
Creating index 'books'
Creating index 'entries'
Indexing 6 'book' objects
Indexing 39 'entry' objects
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/local/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/local/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/elasticsearch/connection/http_urllib3.py", line 255, in perform_request
response = self.pool.urlopen(
File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 798, in urlopen
retries = retries.increment(
File "/usr/local/lib/python3.10/site-packages/urllib3/util/retry.py", line 525, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/local/lib/python3.10/site-packages/urllib3/packages/six.py", line 770, in reraise
raise value
File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
File "/usr/local/lib/python3.10/site-packages/urllib3/connectionpool.py", line 415, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.10/site-packages/urllib3/connection.py", line 244, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/local/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_ch
in documents.py:
from django_elasticsearch_dsl import Document, fields, Index
from django_elasticsearch_dsl.registries import registry
from django.conf import settings
from .models import *
from elasticsearch_dsl import analyzer, tokenizer, connections
autocomplete_analyzer = analyzer('autocomplete_analyzer',
tokenizer=tokenizer('trigram', 'ngram', min_gram=1, max_gram=20),
filter=['lowercase']
)
entry_index=Index('entries')
# See Elasticsearch Indices API reference for available settings
# entry_index.settings(
# number_of_shards=1,
# number_of_replicas=0
# )
# @entry_index.document
@registry.register_document
class EntryDocument(Document):
id = fields.IntegerField(attr='id')
title = fields.TextField(
attr='title',
fields={
'raw': fields.TextField(analyzer=autocomplete_analyzer),
'suggest': fields.CompletionField(),
'key': fields.KeywordField(),
}
)
body = fields.TextField(
attr='body',
fields={
'raw': fields.TextField(analyzer=autocomplete_analyzer),
'suggest': fields.CompletionField(),
}
)
entryauthor = fields.ObjectField(
attr='entryauthor',
properties = {
'id': fields.IntegerField(),
'name': fields.TextField(
attr='name',
fields={
'raw': fields.TextField(),
'key': fields.KeywordField(),
}
) }, )
entryOrigin = fields.ObjectField(
attr = 'entryOrigin',
properties={
'id': fields.IntegerField(),
'country': fields.TextField(
attr='country',
fields={
'raw': fields.KeywordField(),
}
)
}
)
entryCategory = fields.ObjectField(
attr='entryCategory',
properties = {
'id': fields.IntegerField(),
'thecategory': fields.TextField(
attr='thecategory',
fields={
'key': fields.KeywordField(),
'raw': fields.TextField()
}
),
}
)
entryClassification = fields.ObjectField(
attr='entryClassification',
properties = {
'theclass': fields.TextField(
attr='theclass',
fields={
'key': fields.KeywordField(),
}
),
}
)
entryCover = fields.FileField(attr='entryCover')
entryPubDate= fields.DateField(attr='entryPubDate')
bibiliography = fields.TextField(
attr='bibilography',
fields = {
'bibilography': fields.TextField(analyzer='keyword')
}
)
favouriteusers = fields.ObjectField(
attr='favouriteusers',
properties = {
'id': fields.IntegerField(
attr='id',
fields={
'key': fields.KeywordField()
}
),
}
)
viewsCounts = fields.IntegerField(attr='viewsCounts')
source = fields.TextField(
attr='source',
fields={
'raw': fields.TextField(analyzer=autocomplete_analyzer),
'suggest': fields.CompletionField(),
'key': fields.KeywordField(),
}
)
class Index:
name = 'entries'
settings = {
"number_of_shards": 1,
"number_of_replicas": 0,
'max_ngram_diff': 20
}
class Django:
model = entry
fields= []
book_index=Index('books')
@registry.register_document
class BookDocument(Document):
id = fields.IntegerField(attr='id')
name = fields.TextField(
attr='name',
fields={
'raw': fields.TextField(analyzer=autocomplete_analyzer),
'suggest': fields.CompletionField(),
'key': fields.KeywordField(),
}
)
# body = fields.TextField(
# attr='body',
# fields={
# 'raw': fields.TextField(analyzer=autocomplete_analyzer),
# 'suggest': fields.CompletionField(),
# }
# )
author = fields.ObjectField(
attr='author',
properties = {
'id': fields.IntegerField(),
'name': fields.TextField(
attr='name',
fields={
'raw': fields.TextField(),
'key': fields.KeywordField(),
}
) }, )
bookOrigin = fields.ObjectField(
attr = 'bookOrigin',
properties={
'id': fields.IntegerField(),
'country': fields.TextField(
attr='country',
fields={
'raw': fields.KeywordField(),
}
)
}
)
bookCategory = fields.ObjectField(
attr='bookCategory',
properties = {
'id': fields.IntegerField(),
'thecategory': fields.TextField(
attr='thecategory',
fields={
'key': fields.KeywordField(),
'raw': fields.TextField()
}
),
}
)
bookClassification = fields.ObjectField(
attr='bookClassification',
properties = {
'theclass': fields.TextField(
attr='theclass',
fields={
'key': fields.KeywordField(),
}
),
}
)
cover = fields.FileField(attr='cover')
publicationDate= fields.DateField(attr='publicationDate')
class Index:
name = 'books'
settings = {
"number_of_shards": 1,
"number_of_replicas": 0,
'max_ngram_diff': 20
}
class Django:
model = book
fields= []
So how can I solve this bugs?