Indexing in Elasticsearch stops after particular number

Elastic Stack Elasticsearch
1

I am trying to index SQL data from the database using logstash. I am able to index the data properly.
But, after certain number of documents (6191) , Elasticsearch does not have any documets indexed. Logstash keeps on runnig.

 {
 "count" : 6191,
 "_shards" : {
  "total" : 5,
"successful" : 5,
"failed" : 0
  }
}

Here is some config info:

{
 "_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
 "cluster_name" : "elasticcluster-test",
 "nodes" : {
"3stcu47oQbaoNtT-X7NziA" : {
  "timestamp" : 1509551011547,
  "name" : "3stcu47",
  "transport_address" : "10.100.0.0:9300",
  "host" : "0.0.0.0",
  "ip" : "10.10.0.0:9300",
  "roles" : [
    "master",
    "data",
    "ingest"
  ],
  "os" : {
    "timestamp" : 1509551011548,
    "cpu" : {
      "percent" : 18,
      "load_average" : {
        "1m" : 0.38,
        "5m" : 0.48,
        "15m" : 0.57
      }
    },
    "mem" : {
      "total_in_bytes" : 10512769024,
      "free_in_bytes" : 3292909568,
      "used_in_bytes" : 7219859456,
      "free_percent" : 31,
      "used_percent" : 69
    },
    "swap" : {
      "total_in_bytes" : 2011566080,
      "free_in_bytes" : 2011566080,
      "used_in_bytes" : 0
    }
  },
  "process" : {
    "timestamp" : 1509551011548,
    "open_file_descriptors" : 146,
    "max_file_descriptors" : 65536,
    "cpu" : {
      "percent" : 18,
      "total_in_millis" : 4621630
    },
    "mem" : {
      "total_virtual_in_bytes" : 8118427648
    }
  }
}
}
}

Can anyone suggest what is the problem here? I am wiling to index a big dataset, but it stops indexing only after 6K documents.

2

How many cores does this node have? Could you post hot threads for your elasticsearch node?

3

Please show your Logstash config.

4

Info as you asked:

{
   "_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
 },
  "cluster_name" : "elasticcluster-test",
 "nodes" : {
  "3stcu47oQbaoNtT-X7NziA" : {
  "name" : "3stcu47",
  "transport_address" : "10.10.0.0:9300",
  "host" : "10.10.0.0",
  "ip" : "10.10.0.0",
  "version" : "5.4.1",
  "build_hash" : "2cfe0df",
  "total_indexing_buffer" : 535127654,
  "roles" : [
    "master",
    "data",
    "ingest"
  ],
  "settings" : {
    "pidfile" : "/var/run/elasticsearch/elasticsearch.pid",
    "cluster" : {
      "name" : "elasticcluster-test"
    },
    "node" : {
      "name" : "3stcu47"
    },
    "path" : {
      "logs" : "/var/log/elasticsearch",
      "home" : "/usr/share/elasticsearch"
    },
    "default" : {
      "path" : {
        "data" : "/var/lib/elasticsearch",
        "logs" : "/var/log/elasticsearch",
        "conf" : "/etc/elasticsearch"
      }
    },
    "client" : {
      "type" : "node"
    },
    "http" : {
      "type" : {
        "default" : "netty4"
      },
      "port" : "9200",
      "cors" : {
        "allow-origin" : "localhost(:[0-9]+)?/",
        "enabled" : "true"
      }
    },
    "transport" : {
      "type" : {
        "default" : "netty4"
      }
    },
    "network" : {
      "host" : "10.10.0.0"
    }
  },
  "os" : {
    "refresh_interval_in_millis" : 1000,
    "name" : "Linux",
    "arch" : "amd64",
    "version" : "3.16.0-4-amd64",
    "available_processors" : 2,
    "allocated_processors" : 2
  },
  "process" : {
    "refresh_interval_in_millis" : 1000,
    "id" : 8534,
    "mlockall" : false
  },
  "jvm" : {
    "pid" : 8534,
    "version" : "1.8.0_144",
    "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
    "vm_version" : "25.144-b01",
    "vm_vendor" : "Oracle Corporation",
    "start_time_in_millis" : 1509631184804,
    "mem" : {
      "heap_init_in_bytes" : 5368709120,
      "heap_max_in_bytes" : 5351276544,
      "non_heap_init_in_bytes" : 2555904,
      "non_heap_max_in_bytes" : 0,
      "direct_max_in_bytes" : 5351276544
    },
    "gc_collectors" : [
      "ParNew",
      "ConcurrentMarkSweep"
    ],
    "memory_pools" : [
      "Code Cache",
      "Metaspace",
      "Compressed Class Space",
      "Par Eden Space",
      "Par Survivor Space",
      "CMS Old Gen"
    ],
    "using_compressed_ordinary_object_pointers" : "true",
    "input_arguments" : [
      "-Xms5g",
      "-Xmx5g",
      "-XX:+UseConcMarkSweepGC",
      "-XX:CMSInitiatingOccupancyFraction=75",
      "-XX:+UseCMSInitiatingOccupancyOnly",
      "-XX:+DisableExplicitGC",
      "-XX:+AlwaysPreTouch",
      "-Xss1m",
      "-Djava.awt.headless=true",
      "-Dfile.encoding=UTF-8",
      "-Djna.nosys=true",
      "-Djdk.io.permissionsUseCanonicalPath=true",
      "-Dio.netty.noUnsafe=true",
      "-Dio.netty.noKeySetOptimization=true",
      "-Dio.netty.recycler.maxCapacityPerThread=0",
      "-Dlog4j.shutdownHookEnabled=false",
      "-Dlog4j2.disable.jmx=true",
      "-Dlog4j.skipJansi=true",
      "-XX:+HeapDumpOnOutOfMemoryError",
      "-Des.path.home=/usr/share/elasticsearch"
    ]
  },
  "thread_pool" : {
    "force_merge" : {
      "type" : "fixed",
      "min" : 1,
      "max" : 1,
      "queue_size" : -1
    },
    "fetch_shard_started" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 4,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "listener" : {
      "type" : "fixed",
      "min" : 1,
      "max" : 1,
      "queue_size" : -1
    },
    "index" : {
      "type" : "fixed",
      "min" : 2,
      "max" : 2,
      "queue_size" : 200
    },
    "refresh" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 1,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "generic" : {
      "type" : "scaling",
      "min" : 4,
      "max" : 128,
      "keep_alive" : "30s",
      "queue_size" : -1
    },
    "warmer" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 1,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "search" : {
      "type" : "fixed",
      "min" : 4,
      "max" : 4,
      "queue_size" : 1000
    },
    "flush" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 1,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "fetch_shard_store" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 4,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "management" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 5,
      "keep_alive" : "5m",
      "queue_size" : -1
    },
    "get" : {
      "type" : "fixed",
      "min" : 2,
      "max" : 2,
      "queue_size" : 1000
    },
    "bulk" : {
      "type" : "fixed",
      "min" : 2,
      "max" : 2,
      "queue_size" : 200
    },
    "snapshot" : {
      "type" : "scaling",
      "min" : 1,
      "max" : 1,
      "keep_alive" : "5m",
      "queue_size" : -1
    }
  },
  "transport" : {
    "bound_address" : [
      "10.10.0.0:9300"
    ],
    "publish_address" : "10.10.0.0:9300",
    "profiles" : { }
  },
  "http" : {
    "bound_address" : [
      "10.10.0.0:9200"
    ],
    "publish_address" : "10.10.0.0:9200",
    "max_content_length_in_bytes" : 104857600
  },
    ]
  }
}

}
}

Hot threads:

  Hot threads at 2017-11-06T09:28:54.287Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

0.0% (57.5micros out of 500ms) cpu usage by thread 'elasticsearch[3stcu47][[timer]]'
 10/10 snapshots sharing following 2 elements
   java.lang.Thread.sleep(Native Method)
   org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:536)

0.0% (16.1micros out of 500ms) cpu usage by thread 'threadDeathWatcher-2-1'
 10/10 snapshots sharing following 4 elements
   java.lang.Thread.sleep(Native Method)
   io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
   io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
   java.lang.Thread.run(Thread.java:748)
5

My logstash config is a big one. I explain the flow of it here:

 Input:
Start
JDBC: 
   {
   }
filter {
JDBC_streaming
}
Finish

Agrregate Filter Start
{
}
Finish

filter {
Reformat some values 
}

filter {
 Downcase one event
}

filter{
collect one event value
}

filter{
 group some values for events
}

 filter{
 Translate 
 Dictionary
}

  filter{
 mutate
}

 filter{
 Translate 
 Dictionary
}

filter{
mutate
Rename
}

filter{
Translate 
 Dictionary
}

 filter{
mutate
}

 filter{
 Translate 
Dictionary
}

 filter{
mutate
}

 filter {
   ruby code
 }

 filter {
 ruby code
}

 filter {
 ruby code
 }

 filter{
 Translate 
 Dictionary
 }

 filter{
 mutate
}

 filter{
 Translate 
Dictionary
 }

 filter{
mutate
}

  filter{
  Translate 
  Dictionary
   }

            filter{
       mutate
     }

        output {
  stdout { codec => json_lines }
          elasticsearch {
          "hosts" => "localhost"
"index" => "test"
"document_id" =>  " %{s_id}"
     }
       }

This is how it looks like.

6

And when i run Logstash in debug mode

14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.ruby - closing {:plugin=>"LogStash::Filters::Ruby"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.translate - closing {:plugin=>"LogStash::Filters::Translate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.filters.mutate - closing {:plugin=>"LogStash::Filters::Mutate"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.outputs.stdout - closing {:plugin=>"LogStash::Outputs::Stdout"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - closing {:plugin=>"LogStash::Outputs::ElasticSearch"}
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - Stopping sniffer
14:57:38.648 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - Stopping resurrectionist
14:57:38.832 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - Waiting for in use manticore connections
14:57:38.832 [[main]-pipeline-manager] DEBUG logstash.outputs.elasticsearch - Closing adapter #LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter:0x772eaf08
14:57:38.833 [[main]-pipeline-manager] DEBUG logstash.pipeline - Pipeline main has been shutdown
14:57:38.903 [LogStash::Runner] DEBUG logstash.instrument.periodicpoller.os - PeriodicPoller: Stopping
14:57:38.904 [LogStash::Runner] DEBUG logstash.instrument.periodicpoller.jvm - PeriodicPoller: Stopping
14:57:38.904 [LogStash::Runner] DEBUG logstash.instrument.periodicpoller.persistentqueue - PeriodicPoller: Stopping
14:57:38.907 [LogStash::Runner] WARN logstash.agent - stopping pipeline {:id=>"main"}
14:57:38.908 [LogStash::Runner] DEBUG logstash.pipeline - Closing inputs
14:57:38.908 [LogStash::Runner] DEBUG logstash.inputs.jdbc - stopping {:plugin=>"LogStash::Inputs::Jdbc"}
14:57:38.913 [LogStash::Runner] DEBUG logstash.pipeline - Closed inputs

7

How are you running Logstash?

8

Tried both ways:
/usr/share/logstash/bin/logstash --debug -w 1 -f logstash.conf
systemctl start logstash

But same, logstash is running but index is not going beyond 6191 entries. And with debug mode i could see pipeline stops. With systemd it is running continously but not writing to index. I am stuck as i dont want to look for JAVA api option for initial index creation instead want to do it logstash itself.

9

ok what i noticed now with systemctrl way of running logstash, pipeline restarts after this many number. why that is happening with logstash, any idea?

10

Ok so here is what is happening,

I start logstash with systemctl start logstash, it is working fine.

  logstash.service - logstash
 Loaded: loaded (/etc/systemd/system/logstash.service; disabled)
 Active: active (running) since Wed 2017-11-08 13:28:27 CET; 2min 29s ag0
Main PID: 5255 (java)
CGroup: /system.slice/logstash.service
 └─5255 /usr/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=
 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF-
 -XX:+HeapDumpOnOutOfMemoryError -Xmx5g -Xms1g -Xss2048k -Djffi.boot.library.path=/usr/share/logsta...

When it has indexed 6191 entries in Elasticsearch...logstash started with new PID but no more indexing in ES.

logstash.service - logstash
Loaded: loaded (/etc/systemd/system/logstash.service; disabled)
Active: active (running) since Wed 2017-11-08 13:30:57 CET; 10s ago
Main PID: 5320 (java)
CGroup: /system.slice/logstash.service
 └─5320 /usr/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=7
 -XX:+UseCMSInitiatingOccupancyOnly -XX:+DisableExplicitGC -Djava.awt.headless=true -Dfile.encoding=UTF-8
-XX:+HeapDumpOnOutOfMemoryError -Xmx5g -Xms1g -Xss2048k -Djffi.boot.library.path=/usr/share/logsta...
 Nov 08 13:30:57 vm-logstash01-test systemd[1]: Started logstash
 Nov 08 13:31:07 vm-logstash01-test logstash[5320]: Sending Logstash's logs to /var/log/logstash which is now configured via log4j2.properties

And this process goes on but no more indexing of documents index, why this behavriou is happing, any idea?

11

logstash.service holdoff time over, scheduling restart....this is what is happening when its changing process id.

I have no scheduling parameters in my conf file.

12

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.