TransportClient causes abnormal system load


(Jian Lin) #1

es cluster version: 2.3.3
es client version: 2.3.3
machine: centos6 8GB RAM 4 core cpu

Sometimes there is no regular pattern of sudden increase in load, but io and cpu used is normal.Look the picture,on the left is the load value.

When I use the transport client, I do not use it for any es operations. The system load will rise inexplicably.
I found when i remove the transport client instantiation code, the system load is normal.
Here is my transport client instance code.

  @Bean
    public TransportClient transportClient() {
        String clusterName = env.getProperty("elasticsearch.cluster.name");
        String userName = env.getProperty("elasticsearch.cluster.userName");
        String password = env.getProperty("elasticsearch.cluster.password");
        Builder builder = Settings.settingsBuilder()
            .put("client.transport.sniff", true)
            .put("client.transport.ping_timeout", "120s")
            .put("cluster.name", clusterName)
            .put("shield.user", MessageFormat.format("{0}:{1}", userName, password));
        Settings settings = builder.build();
        try {
            TransportClient client = 
           TransportClient.builder().addPlugin(ShieldPlugin.class).settings(settings).build();
            String hosts = env.getProperty("elasticsearch.cluster.hostsAndPorts");
            if (org.apache.commons.lang.StringUtils.isBlank(hosts)) {
                throw new IllegalArgumentException("Elasticsearch cluster hosts and ports is empty");
            }
            String[] hostArr = org.apache.commons.lang.StringUtils.split(hosts, ",");
            for (String node : hostArr) {
                String ip = node.split(":")[0].trim();
                int port = 9300;
                if (node.split(":").length > 1) {
                    port = Integer.parseInt(node.split(":")[1]);
                }
                client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), port));
            }
            if (client.connectedNodes().size() == 0) {
                LOG.error("Has not usable ES nodes to connect.");
                throw new IOException("ES connect fail.");
            }
            return client;
        } catch (Exception e) {
            LOG.error("++> Init elasticsearch client failed", e);
            System.exit(1);
        }
        return null;
    }

(Jian Lin) #2

here is my thread stack:

"elasticsearch[Razor Fist][transport_client_boss][T#1]{New I/O boss #9}" #51 daemon prio=5 os_prio=0 tid=0x00007ff080809800 nid=0x388c runnable [0x00007ff0686e1000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x00000007018742f0> (a sun.nio.ch.Util$3)
	- locked <0x00000007018742d8> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000007016ca2a8> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
	at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42)
	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

"elasticsearch[Razor Fist][transport_client_worker][T#8]{New I/O worker #8}" #49 daemon prio=5 os_prio=0 tid=0x00007ff0e4db5000 nid=0x388b runnable [0x00007ff0687e2000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x0000000701b75b78> (a sun.nio.ch.Util$3)
	- locked <0x0000000701b75b60> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000007016ca4a0> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

"elasticsearch[Razor Fist][transport_client_worker][T#7]{New I/O worker #7}" #48 daemon prio=5 os_prio=0 tid=0x00007ff0e4db3800 nid=0x388a runnable [0x00007ff0688e3000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x0000000701d0fb50> (a sun.nio.ch.Util$3)
	- locked <0x0000000701d0fb38> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000007016ca5c0> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)

"elasticsearch[Razor Fist][transport_client_worker][T#6]{New I/O worker #6}" #47 daemon prio=5 os_prio=0 tid=0x00007ff0e4db1800 nid=0x3889 runnable [0x00007ff0689e4000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
	at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
	at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
	at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
	- locked <0x0000000701dfb210> (a sun.nio.ch.Util$3)
	- locked <0x0000000701dfb1f8> (a java.util.Collections$UnmodifiableSet)
	- locked <0x00000007016ca650> (a sun.nio.ch.EPollSelectorImpl)
	at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
	at org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:68)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.select(AbstractNioSelector.java:434)
	at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:212)
	at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89)
	at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178)
	at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
	at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)


...


(system) #3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.