@Artem_Shelkovnikov @maryna.cherniavska
I switched log level to debug and no errors listed before the container just stops (the last items in the log are the single domain crawl activity):
[2022-06-16T15:56:14.163+00:00][8][5060][es][DEBUG]: {
"request": {
"url": "https://es01:9200/.ent-search-actastic-app_search_accounts_v11/_search?request_cache=true",
"method": "get",
"headers": {
"Authorization": "[FILTERED]",
"Content-Type": "application/json",
"x-elastic-product-origin": "enterprise-search",
"User-Agent": "Faraday v1.8.0"
},
"params": null,
"body": "{\"query\":{\"bool\":{}},\"sort\":[\"_doc\"],\"size\":1,\"from\":0,\"seq_no_primary_term\":true}"
},
"response": {
"status": 200,
"headers": {
"x-elastic-product": "Elasticsearch",
"content-type": "application/json",
"content-length": "749"
},
"body": "{\"took\":2,\"timed_out\":false,\"_shards\":{\"total\":1,\"successful\":1,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":{\"value\":1,\"relation\":\"eq\"},\"max_score\":null,\"hits\":[{\"_index\":\".ent-search-actastic-app_search_accounts_v11\",\"_id\":\"62aa385b0ba5b28255fc8fd6\",\"_seq_no\":0,\"_primary_term\":1,\"_score\":null,\"_source\":{\"id\":\"62aa385b0ba5b28255fc8fd6\",\"created_at\":\"2022-06-15T19:51:55Z\",\"updated_at\":\"2022-06-15T19:51:55Z\",\"limit_overrides\":{},\"metered_plan_limit_overrides\":{},\"seeding_sample_engine\":null,\"onboarding_state\":\"create_engine\",\"name\":\"enterprise_search\",\"authorization_strategies\":null,\"telemetry_last_sent_at\":null,\"api_log_disabled_at\":null,\"analytics_log_disabled_at\":null,\"crawler_log_disabled_at\":null,\"audit_log_disabled_at\":null},\"sort\":[0]}]}}"
},
"duration": 6.6,
"stack": [
"lib/actastic/schema.class:55:in `search'",
"lib/actastic/relation.class:538:in `block in search'",
"lib/apm_helpers.class:52:in `es_action_instrument'",
"lib/apm_helpers.class:57:in `actastic_instrument'",
"lib/actastic/relation.class:518:in `instrument'",
"lib/actastic/relation.class:537:in `search'",
"lib/actastic/relation.class:225:in `find_each'",
"lib/actastic/relation.class:307:in `to_a'",
"lib/actastic/relation.class:307:in `load'",
"lib/actastic/relation.class:302:in `to_a'",
"lib/actastic/relation.class:323:in `each'",
"lib/actastic/relation.class:216:in `first'",
"lib/actastic/relation.class:216:in `first_with_limit'",
"lib/actastic/actastic_record/class_methods.class:391:in `first'",
"shared_togo/lib/shared_togo/crawler/event_log_formatter.class:8:in `skip_ingest?'",
"shared_togo/lib/shared_togo/events/event_log_formatter.class:9:in `call'",
"loco_moco/lib/crawler/loco_moco/api/config.class:14:in `<<'",
"crawler/lib/crawler/api/config.class:322:in `output_event'",
"loco_moco/lib/crawler/loco_moco/api/config.class:88:in `output_event'",
"crawler/lib/crawler/event_generator.class:330:in `log'",
"crawler/lib/crawler/event_generator.class:305:in `log_event'",
"crawler/lib/crawler/event_generator.class:296:in `log_crawl_event'",
"crawler/lib/crawler/event_generator.class:292:in `log_url_event'",
"crawler/lib/crawler/event_generator.class:175:in `url_discover'",
"crawler/lib/crawler/coordinator.class:609:in `check_discovered_url'",
"crawler/lib/crawler/coordinator.class:470:in `block in add_urls_to_backlog'",
"crawler/lib/crawler/coordinator.class:456:in `each'",
"crawler/lib/crawler/coordinator.class:456:in `add_urls_to_backlog'",
"crawler/lib/crawler/coordinator.class:360:in `extract_and_enqueue_html_links'",
"crawler/lib/crawler/coordinator.class:321:in `extract_and_enqueue_links'",
"crawler/lib/crawler/coordinator.class:283:in `block in process_crawl_result'",
"crawler/lib/crawler/coordinator.class:283:in `process_crawl_result'",
"crawler/lib/crawler/coordinator.class:257:in `execute_crawl_task'",
"crawler/lib/crawler/coordinator.class:242:in `block in run_crawl_loop'"
]
}
[2022-06-16T15:56:14.168+00:00][8][5116][es][DEBUG]: {
"request": {
"url": "https://es01:9200/.ent-search-actastic-app_search_accounts_v11/_search?request_cache=true",
"method": "get",
"headers": {
"Authorization": "[FILTERED]",
"Content-Type": "application/json",
"x-elastic-product-origin": "enterprise-search",
"User-Agent": "Faraday v1.8.0"
},
"params": null,
"body": "{\"query\":{\"bool\":{}},\"sort\":[\"_doc\"],\"size\":1,\"from\":0,\"seq_no_primary_term\":true}"
},
"response": {
"status": 200,
"headers": {
"x-elastic-product": "Elasticsearch",
"content-type": "application/json",
"content-length": "749"
},
"body": "{\"took\":0,\"timed_out\":false,\"_shards\":{\"total\":1,\"successful\":1,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":{\"value\":1,\"relation\":\"eq\"},\"max_score\":null,\"hits\":[{\"_index\":\".ent-search-actastic-app_search_accounts_v11\",\"_id\":\"62aa385b0ba5b28255fc8fd6\",\"_seq_no\":0,\"_primary_term\":1,\"_score\":null,\"_source\":{\"id\":\"62aa385b0ba5b28255fc8fd6\",\"created_at\":\"2022-06-15T19:51:55Z\",\"updated_at\":\"2022-06-15T19:51:55Z\",\"limit_overrides\":{},\"metered_plan_limit_overrides\":{},\"seeding_sample_engine\":null,\"onboarding_state\":\"create_engine\",\"name\":\"enterprise_search\",\"authorization_strategies\":null,\"telemetry_last_sent_at\":null,\"api_log_disabled_at\":null,\"analytics_log_disabled_at\":null,\"crawler_log_disabled_at\":null,\"audit_log_disabled_at\":null},\"sort\":[0]}]}}"
},
"duration": 20.9,
"stack": [
"lib/actastic/schema.class:55:in `search'",
"lib/actastic/relation.class:538:in `block in search'",
"lib/apm_helpers.class:52:in `es_action_instrument'",
"lib/apm_helpers.class:57:in `actastic_instrument'",
"lib/actastic/relation.class:518:in `instrument'",
"lib/actastic/relation.class:537:in `search'",
"lib/actastic/relation.class:225:in `find_each'",
"lib/actastic/relation.class:307:in `to_a'",
"lib/actastic/relation.class:307:in `load'",
"lib/actastic/relation.class:302:in `to_a'",
"lib/actastic/relation.class:323:in `each'",
"lib/actastic/relation.class:216:in `first'",
"lib/actastic/relation.class:216:in `first_with_limit'",
"lib/actastic/actastic_record/class_methods.class:391:in `first'",
"shared_togo/lib/shared_togo/crawler/event_log_formatter.class:8:in `skip_ingest?'",
"shared_togo/lib/shared_togo/events/event_log_formatter.class:9:in `call'",
"loco_moco/lib/crawler/loco_moco/api/config.class:14:in `<<'",
"crawler/lib/crawler/api/config.class:322:in `output_event'",
"loco_moco/lib/crawler/loco_moco/api/config.class:88:in `output_event'",
"crawler/lib/crawler/event_generator.class:330:in `log'",
"crawler/lib/crawler/event_generator.class:305:in `log_event'",
"crawler/lib/crawler/event_generator.class:296:in `log_crawl_event'",
"crawler/lib/crawler/event_generator.class:292:in `log_url_event'",
"crawler/lib/crawler/event_generator.class:175:in `url_discover'",
"crawler/lib/crawler/event_generator.class:189:in `url_discover_denied'",
"crawler/lib/crawler/coordinator.class:604:in `check_discovered_url'",
"crawler/lib/crawler/coordinator.class:470:in `block in add_urls_to_backlog'",
"crawler/lib/crawler/coordinator.class:456:in `each'",
"crawler/lib/crawler/coordinator.class:456:in `add_urls_to_backlog'",
"crawler/lib/crawler/coordinator.class:360:in `extract_and_enqueue_html_links'",
"crawler/lib/crawler/coordinator.class:321:in `extract_and_enqueue_links'",
"crawler/lib/crawler/coordinator.class:283:in `block in process_crawl_result'",
"crawler/lib/crawler/coordinator.class:283:in `process_crawl_result'",
"crawler/lib/crawler/coordinator.class:257:in `execute_crawl_task'",
"crawler/lib/crawler/coordinator.class:242:in `block in run_crawl_loop'"
]
}
[2022-06-16T15:56:14.183+00:00][8][5068][es][DEBUG]: {
"request": {
"url": "https://es01:9200/enterprise-search-engine-creditkarmapub/_bulk",
"method": "post",
"headers": {
"Authorization": "[FILTERED]",
"Content-Type": "application/json",
"x-elastic-product-origin": "enterprise-search",
"User-Agent": "Faraday v1.8.0"
},
"params": null,
"body": "removed to save space in this message"}\n"
},
"response": {
"status": 200,
"headers": {
"x-elastic-product": "Elasticsearch",
"content-type": "application/json",
"content-length": "260"
},
"body": "{\"took\":45,\"errors\":false,\"items\":[{\"index\":{\"_index\":\".ent-search-engine-documents-creditkarmapub\",\"_id\":\"62ab528a329d3c9a199894ed\",\"_version\":1,\"result\":\"created\",\"_shards\":{\"total\":2,\"successful\":2,\"failed\":0},\"_seq_no\":69,\"_primary_term\":9,\"status\":201}}]}"
},
"duration": 52.6,
"stack": [
"lib/swiftype/document_storage/backends/managed_index_backend.class:74:in `bulk_action'",
"lib/swiftype/document_storage/backends/managed_index_backend.class:26:in `block in save_documents'",
"lib/swiftype/document_storage/backends/managed_index_backend.class:21:in `each'",
"lib/swiftype/document_storage/backends/managed_index_backend.class:21:in `save_documents'",
"lib/swiftype/documents.class:94:in `save_documents'",
"app/models/search_index/documents_concern.class:51:in `save_documents'",
"lib/abstract_upsert_document_service.class:46:in `upsert'",
"loco_moco/lib/crawler/output_sink/app_search.class:272:in `upsert_app_search_document'",
"loco_moco/lib/crawler/output_sink/app_search.class:156:in `process_success'",
"loco_moco/lib/crawler/output_sink/app_search.class:74:in `process_crawl_result'",
"loco_moco/lib/crawler/output_sink/app_search.class:34:in `block in write'",
"loco_moco/lib/crawler/output_sink/app_search.class:33:in `write'",
"crawler/lib/crawler/coordinator.class:431:in `output_crawl_result'",
"crawler/lib/crawler/coordinator.class:306:in `process_crawl_result'",
"crawler/lib/crawler/coordinator.class:257:in `execute_crawl_task'",
"crawler/lib/crawler/coordinator.class:242:in `block in run_crawl_loop'"
]
}
[2022-06-16T15:56:14.736+00:00][8][5068][crawler][INFO]: [crawl:62aa5c5b0ba5b27abefc8fee] [primary] Processed crawl results from the page 'https://www.creditkarma.com/resources' via the app_search output. Outcome: success. Message: Indexed the document into App Search with doc_id=62ab528a329d3c9a199894ed.
Docker Ram is set to 8gb with single container limits set to 1gb for each container instance:
portion of my docker-compose.yml:
...
environment:
- "secret_management.encryption_keys=['REDACTED']"
- "allow_es_settings_modification=true"
- "elasticsearch.host='https://es01:9200'"
- "elasticsearch.username='elastic'"
- "elasticsearch.password='REDACTED'"
- "elasticsearch.ssl.enabled=true"
- "elasticsearch.ssl.certificate_authority='/usr/share/enterprise-search/config/certs/ca/ca.crt'"
- "elasticsearch.ssl.certificate='/usr/share/enterprise-search/config/certs/es01/es01.crt'"
- "elasticsearch.ssl.key='/usr/share/enterprise-search/config/certs/es01/es01.key'"
- "elasticsearch.ssl.verify=true"
- "kibana.host='http://kibana:5601'"
- "kibana.external_url='http://localhost:5601'"
- "secret_session_key=REDACTED"
- "secret_management.enforce_valid_encryption_keys=false"
- "ent_search.ssl.enabled=false"
- "ent_search.external_url='http://localhost:3002'"
- "ent_search.listen_host=0.0.0.0"
- "ent_search.listen_port=3002"
- "log_level=debug"
mem_limit: 1gb
...
@maryna.cherniavska Good call on the state inspect - the information from the Enterprise Search container instance after it exits itself - which it looks like it is an out of memory issue OOMKilled = true:
"State": {
"Status": "exited",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": true,
"Dead": false,
"Pid": 0,
"ExitCode": 137,
"Error": "",
"StartedAt": "2022-06-16T16:35:29.400521377Z",
"FinishedAt": "2022-06-16T16:37:13.389957092Z",
"Health": {
"Status": "unhealthy",
"FailingStreak": 0,
"Log": [
{
"Start": "2022-06-16T16:36:22.459438555Z",
"End": "2022-06-16T16:36:22.965434055Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2022-06-16T16:36:32.97288792Z",
"End": "2022-06-16T16:36:33.875854587Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2022-06-16T16:36:43.893081217Z",
"End": "2022-06-16T16:36:44.745158676Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2022-06-16T16:36:54.758904833Z",
"End": "2022-06-16T16:36:56.311644042Z",
"ExitCode": 0,
"Output": ""
},
{
"Start": "2022-06-16T16:37:06.319005422Z",
"End": "2022-06-16T16:37:09.42528309Z",
"ExitCode": 0,
"Output": ""
}
]
}
},
I changed the mem_limit value from 1gb to 4gb and now it seems the container instance is stable and not killing itself.
Thank you for the helpful hints!