Merge pull request hazelcast#7514 from pveentjer/feature/3.x/client-m…

…etrics

Adds Metrics and PerformanceLog to the client.

hazelcast-client/src/main/java/com/hazelcast/client/connection/nio/ClientConnectionManagerImpl.java

@@ -148,11 +148,14 @@ protected void initializeSelectors(HazelcastClientInstanceImpl client) {
                 client.getName() + ".ClientInSelector",
                 loggingService.getLogger(NonBlockingIOThread.class),
                 OUT_OF_MEMORY_HANDLER);
+        client.getMetricsRegistry().scanAndRegister(inputThread, "tcp." + inputThread.getName());
+
         outputThread = new ClientNonBlockingOutputThread(
                 client.getThreadGroup(),
                 client.getName() + ".ClientOutSelector",
                 loggingService.getLogger(ClientNonBlockingOutputThread.class),
                 OUT_OF_MEMORY_HANDLER);
+        client.getMetricsRegistry().scanAndRegister(outputThread, "tcp." + outputThread.getName());
     }
 
     private SocketInterceptor initSocketInterceptor(SocketInterceptorConfig sic) {

hazelcast-client/src/main/java/com/hazelcast/client/impl/HazelcastClientInstanceImpl.java

@@ -75,6 +75,13 @@
 import com.hazelcast.executor.impl.DistributedExecutorService;
 import com.hazelcast.instance.BuildInfoProvider;
 import com.hazelcast.internal.properties.GroupProperty;
+import com.hazelcast.instance.HazelcastThreadGroup;
+import com.hazelcast.internal.metrics.ProbeLevel;
+import com.hazelcast.internal.metrics.impl.MetricsRegistryImpl;
+import com.hazelcast.internal.monitors.ConfigPropertiesPlugin;
+import com.hazelcast.internal.monitors.MetricsPlugin;
+import com.hazelcast.internal.monitors.PerformanceMonitor;
+import com.hazelcast.internal.monitors.SystemPropertiesPlugin;
 import com.hazelcast.internal.serialization.InternalSerializationService;
 import com.hazelcast.logging.ILogger;
 import com.hazelcast.logging.LoggingService;
@@ -116,6 +123,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 
+import static java.lang.System.currentTimeMillis;
+
 public class HazelcastClientInstanceImpl implements HazelcastInstance, SerializationServiceSupport {
 
     private static final AtomicInteger CLIENT_ID = new AtomicInteger();
@@ -142,7 +151,9 @@ public class HazelcastClientInstanceImpl implements HazelcastInstance, Serializa
     private final Credentials credentials;
     private final DiscoveryService discoveryService;
     private final LoggingService loggingService;
-    private SerializationService serializationService;
+    private final MetricsRegistryImpl metricsRegistry;
+    private final PerformanceMonitor performanceMonitor;
+    private final SerializationService serializationService;
 
     public HazelcastClientInstanceImpl(ClientConfig config,
                                        ClientConnectionManagerFactory clientConnectionManagerFactory,
@@ -165,6 +176,8 @@ public HazelcastClientInstanceImpl(ClientConfig config,
         threadGroup = new ThreadGroup(instanceName);
         lifecycleService = new LifecycleServiceImpl(this);
         clientProperties = new ClientProperties(config);
+
+        metricsRegistry = initMetricsRegistry();
         serializationService = clientExtension.createSerializationService((byte) -1);
         proxyManager = new ProxyManager(this);
         executionService = initExecutionService();
@@ -180,9 +193,23 @@ public HazelcastClientInstanceImpl(ClientConfig config,
         userContext = new ConcurrentHashMap<String, Object>();
         nearCacheManager = clientExtension.createNearCacheManager();
 
+        performanceMonitor = initPerformanceMonitor(config);
+
         proxyManager.init(config);
     }
 
+    private PerformanceMonitor initPerformanceMonitor(ClientConfig config) {
+        String name = "performance-client-" + id + "-" + currentTimeMillis();
+        ILogger logger = loggingService.getLogger(PerformanceMonitor.class);
+        HazelcastThreadGroup hzThreadGroup = new HazelcastThreadGroup(getName(), logger, config.getClassLoader());
+        return new PerformanceMonitor(name, logger, hzThreadGroup, clientProperties);
+    }
+
+    private MetricsRegistryImpl initMetricsRegistry() {
+        ProbeLevel probeLevel = clientProperties.getEnum(GroupProperty.PERFORMANCE_METRICS_LEVEL, ProbeLevel.class);
+        return new MetricsRegistryImpl(loggingService.getLogger(MetricsRegistryImpl.class), probeLevel);
+    }
+
     private Collection<AddressProvider> createAddressProviders(AddressProvider externalAddressProvider) {
         ClientNetworkConfig networkConfig = getClientConfig().getNetworkConfig();
         final ClientAwsConfig awsConfig = networkConfig.getAwsConfig();
@@ -319,6 +346,18 @@ public void start() {
         loadBalancer.init(getCluster(), config);
         partitionService.start();
         clientExtension.afterStart(this);
+
+        performanceMonitor.start();
+        performanceMonitor.register(
+                new ConfigPropertiesPlugin(loggingService.getLogger(ConfigPropertiesPlugin.class), clientProperties));
+        performanceMonitor.register(
+                new SystemPropertiesPlugin(loggingService.getLogger(SystemPropertiesPlugin.class)));
+        performanceMonitor.register(
+                new MetricsPlugin(loggingService.getLogger(MetricsPlugin.class), metricsRegistry, clientProperties));
+    }
+
+    public MetricsRegistryImpl getMetricsRegistry() {
+        return metricsRegistry;
     }
 
     @Override
@@ -600,6 +639,8 @@ public void doShutdown() {
         if (discoveryService != null) {
             discoveryService.destroy();
         }
+        metricsRegistry.shutdown();
+        performanceMonitor.shutdown();
     }
 
 }

hazelcast-client/src/main/java/com/hazelcast/client/spi/impl/ClientInvocationServiceSupport.java

@@ -28,6 +28,8 @@
 import com.hazelcast.client.spi.ClientPartitionService;
 import com.hazelcast.client.spi.EventHandler;
 import com.hazelcast.client.spi.impl.listener.ClientListenerServiceImpl;
+import com.hazelcast.internal.metrics.Probe;
+import com.hazelcast.internal.metrics.ProbeLevel;
 import com.hazelcast.logging.ILogger;
 import com.hazelcast.nio.ClassLoaderUtil;
 import com.hazelcast.nio.Connection;
@@ -58,6 +60,8 @@ abstract class ClientInvocationServiceSupport implements ClientInvocationService
     protected ClientListenerServiceImpl clientListenerService;
     protected final ILogger invocationLogger;
     private ResponseThread responseThread;
+
+    @Probe(name = "pendingCalls", level = ProbeLevel.MANDATORY)
     private ConcurrentMap<Long, ClientInvocation> callIdMap
             = new ConcurrentHashMap<Long, ClientInvocation>();
 
@@ -70,6 +74,8 @@ public ClientInvocationServiceSupport(HazelcastClientInstanceImpl client) {
         int maxAllowedConcurrentInvocations = client.getClientProperties().getInteger(MAX_CONCURRENT_INVOCATIONS);
         callIdSequence = new CallIdSequence.CallIdSequenceFailFast(maxAllowedConcurrentInvocations);
         invocationLogger = client.getLoggingService().getLogger(ClientInvocationService.class);
+
+        client.getMetricsRegistry().scanAndRegister(this, "invocations");
     }
 
     @Override

hazelcast/src/main/java/com/hazelcast/instance/GroupProperties.java

@@ -18,6 +18,7 @@
 
 import com.hazelcast.config.Config;
 import com.hazelcast.internal.properties.GroupProperty;
+import com.hazelcast.internal.monitors.PerformanceMonitor;
 
 /**
  * Container for configured Hazelcast properties.
@@ -37,16 +38,16 @@ public class GroupProperties extends HazelcastProperties {
     @Deprecated
     public static final String PROP_HEALTH_MONITORING_DELAY_SECONDS = GroupProperty.HEALTH_MONITORING_DELAY_SECONDS.getName();
     @Deprecated
-    public static final String PROP_PERFORMANCE_MONITOR_ENABLED = GroupProperty.PERFORMANCE_MONITOR_ENABLED.getName();
+    public static final String PROP_PERFORMANCE_MONITOR_ENABLED = PerformanceMonitor.ENABLED.getName();
     @Deprecated
     public static final String PROP_PERFORMANCE_MONITOR_MAX_ROLLED_FILE_SIZE_MB
-            = GroupProperty.PERFORMANCE_MONITOR_MAX_ROLLED_FILE_SIZE_MB.getName();
+            = PerformanceMonitor.MAX_ROLLED_FILE_SIZE_MB.getName();
     @Deprecated
     public static final String PROP_PERFORMANCE_MONITOR_MAX_ROLLED_FILE_COUNT
-            = GroupProperty.PERFORMANCE_MONITOR_MAX_ROLLED_FILE_COUNT.getName();
+            = PerformanceMonitor.MAX_ROLLED_FILE_COUNT.getName();
     @Deprecated
     public static final String PROP_PERFORMANCE_MONITOR_HUMAN_FRIENDLY_FORMAT
-            = GroupProperty.PERFORMANCE_MONITOR_HUMAN_FRIENDLY_FORMAT.getName();
+            = PerformanceMonitor.HUMAN_FRIENDLY_FORMAT.getName();
     @Deprecated
     public static final String PROP_PHONE_HOME_ENABLED = GroupProperty.PHONE_HOME_ENABLED.getName();
     @Deprecated

hazelcast/src/main/java/com/hazelcast/internal/monitors/HealthMonitor.java

@@ -120,7 +120,7 @@ private HealthMonitorThread(int delaySeconds) {
                     node.getHazelcastThreadGroup().getThreadNamePrefix("HealthMonitor"));
             setDaemon(true);
             this.delaySeconds = delaySeconds;
-            this.performanceLogHint = node.getGroupProperties().getBoolean(GroupProperty.PERFORMANCE_MONITOR_ENABLED);
+            this.performanceLogHint = node.getGroupProperties().getBoolean(PerformanceMonitor.ENABLED);
         }
 
         @Override
@@ -167,7 +167,7 @@ private void logPerformanceMonitorHint() {
 
             logger.info(String.format("The HealthMonitor has detected a high load on the system. For more detailed information,%s"
                             + "enable the PerformanceMonitor by adding the property -D%s=true",
-                    LINE_SEPARATOR, GroupProperty.PERFORMANCE_MONITOR_ENABLED));
+                    LINE_SEPARATOR, PerformanceMonitor.ENABLED));
         }
     }
 

hazelcast/src/main/java/com/hazelcast/internal/monitors/InvocationPlugin.java

@@ -17,7 +17,7 @@
 package com.hazelcast.internal.monitors;
 
 import com.hazelcast.internal.properties.GroupProperties;
-import com.hazelcast.internal.properties.GroupProperty;
+import com.hazelcast.internal.properties.HazelcastProperty;
 import com.hazelcast.logging.ILogger;
 import com.hazelcast.spi.impl.NodeEngineImpl;
 import com.hazelcast.spi.impl.operationservice.InternalOperationService;
@@ -27,20 +27,35 @@
 import com.hazelcast.util.Clock;
 import com.hazelcast.util.ItemCounter;
 
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_INVOCATION_SAMPLE_PERIOD_SECONDS;
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_INVOCATION_SLOW_THRESHOLD_SECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
 
 /**
  * A {@link PerformanceMonitorPlugin} that displays all invocations that have been executing for some time.
  *
  * It will display the current invocations.
  *
  * But it will also display the history. E.g. if a entry processor has been running for 5 minutes and
- * the {@link GroupProperty#PERFORMANCE_MONITOR_INVOCATION_SAMPLE_PERIOD_SECONDS} is set to
- * 1 minute, then there will be 5 samples for that given invocation. This is useful to track which operations have
- * been slow over a longer period of time.
+ * the {@link #SAMPLE_PERIOD_SECONDS is set to 1 minute, then there will be 5 samples for that given invocation.
+ * This is useful to track which operations have been slow over a longer period of time.
  */
 public class InvocationPlugin extends PerformanceMonitorPlugin {
+
+    /**
+     * The sample period in seconds for the {@link InvocationPlugin}.
+     *
+     * If set to 0, the plugin is disabled.
+     */
+    public static final HazelcastProperty SAMPLE_PERIOD_SECONDS
+            = new HazelcastProperty("hazelcast.performance.monitor.invocation.sample.period.seconds", 0, SECONDS);
+
+    /**
+     * The threshold in seconds for the {@link InvocationPlugin} to consider an invocation to be slow.
+     */
+    public static final HazelcastProperty SLOW_THRESHOLD_SECONDS
+            = new HazelcastProperty("hazelcast.performance.monitor.invocation.slow.threshold.seconds", 5, SECONDS);
+
+
+
     private final InvocationRegistry invocationRegistry;
     private final long samplePeriodMillis;
     private final long thresholdMillis;
@@ -53,8 +68,8 @@ public InvocationPlugin(NodeEngineImpl nodeEngine) {
         this.invocationRegistry = ((OperationServiceImpl) operationService).getInvocationRegistry();
         this.logger = nodeEngine.getLogger(PendingInvocationsPlugin.class);
         GroupProperties props = nodeEngine.getGroupProperties();
-        this.samplePeriodMillis = props.getMillis(PERFORMANCE_MONITOR_INVOCATION_SAMPLE_PERIOD_SECONDS);
-        this.thresholdMillis = props.getMillis(PERFORMANCE_MONITOR_INVOCATION_SLOW_THRESHOLD_SECONDS);
+        this.samplePeriodMillis = props.getMillis(SAMPLE_PERIOD_SECONDS);
+        this.thresholdMillis = props.getMillis(SLOW_THRESHOLD_SECONDS);
     }
 
     @Override

hazelcast/src/main/java/com/hazelcast/internal/monitors/MetricsPlugin.java

@@ -19,29 +19,42 @@
 import com.hazelcast.internal.metrics.MetricsRegistry;
 import com.hazelcast.internal.metrics.renderers.ProbeRenderer;
 import com.hazelcast.internal.properties.HazelcastProperties;
+import com.hazelcast.internal.properties.HazelcastProperty;
 import com.hazelcast.logging.ILogger;
 import com.hazelcast.spi.impl.NodeEngineImpl;
 
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_METRICS_PERIOD_SECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
 
 /**
  * A {@link PerformanceMonitorPlugin} that displays the content of the {@link MetricsRegistry}.
  */
 public class MetricsPlugin extends PerformanceMonitorPlugin {
+    /**
+     * The period in seconds the PerformanceMonitor MetricsPlugin runs.
+     *
+     * The MetricsPlugin does nothing more than frequently writing the content of the MetricsRegistry
+     * to the logfile. For debugging purposes make sure the {@link #PERFORMANCE_METRICS_LEVEL} is set to debug.
+     *
+     * This plugin is very cheap to use.
+     *
+     * If set to 0, the plugin is disabled.
+     */
+    public static final HazelcastProperty PERIOD_SECONDS
+            = new HazelcastProperty("hazelcast.performance.monitor.metrics.period.seconds", 60, SECONDS);
 
     private final MetricsRegistry metricsRegistry;
     private final long periodMillis;
     private final ILogger logger;
     private final ProbeRendererImpl probeRenderer = new ProbeRendererImpl();
 
     public MetricsPlugin(NodeEngineImpl nodeEngine) {
-        this(nodeEngine.getMetricsRegistry(), nodeEngine.getLogger(MetricsPlugin.class), nodeEngine.getNode().groupProperties);
+        this(nodeEngine.getLogger(MetricsPlugin.class), nodeEngine.getMetricsRegistry(), nodeEngine.getNode().groupProperties);
     }
 
-    public MetricsPlugin(MetricsRegistry metricsRegistry, ILogger logger, HazelcastProperties properties) {
+    public MetricsPlugin(ILogger logger, MetricsRegistry metricsRegistry, HazelcastProperties properties) {
         this.metricsRegistry = metricsRegistry;
         this.logger = logger;
-        this.periodMillis = properties.getMillis(PERFORMANCE_MONITOR_METRICS_PERIOD_SECONDS);
+        this.periodMillis = properties.getMillis(PERIOD_SECONDS);
     }
 
     @Override

hazelcast/src/main/java/com/hazelcast/internal/monitors/OverloadedConnectionsPlugin.java

@@ -17,9 +17,11 @@
 package com.hazelcast.internal.monitors;
 
 import com.hazelcast.internal.properties.GroupProperties;
+import com.hazelcast.internal.properties.HazelcastProperty;
+import com.hazelcast.nio.OutboundFrame;
+
 import com.hazelcast.logging.ILogger;
 import com.hazelcast.nio.ConnectionManager;
-import com.hazelcast.nio.OutboundFrame;
 import com.hazelcast.nio.Packet;
 import com.hazelcast.nio.tcp.TcpIpConnection;
 import com.hazelcast.nio.tcp.TcpIpConnectionManager;
@@ -37,11 +39,9 @@
 import java.util.Random;
 import java.util.Set;
 
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_PERIOD_SECONDS;
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_SAMPLES;
-import static com.hazelcast.internal.properties.GroupProperty.PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_THRESHOLD;
 import static java.lang.Math.min;
 import static java.util.Collections.emptySet;
+import static java.util.concurrent.TimeUnit.SECONDS;
 
 /**
  * The OverloadedConnectionsPlugin checks all the connections and samples the content of the packet
@@ -52,6 +52,34 @@
  * plugin is disabled by default.
  */
 public class OverloadedConnectionsPlugin extends PerformanceMonitorPlugin {
+
+    /**
+     * The period in seconds the PerformanceMonitor OverloadedConnectionPlugin runs.
+     *
+     * With the OverloadedConnectionsPlugin one can see what is going on inside a connection with a huge
+     * number of pending packets. It makes use of sampling to give some impression of the content.
+     *
+     * This plugin can be very expensive to use and should only be used as a debugging aid; should not be
+     * used in production due to the fact that packets could be deserialized.
+     *
+     * If set to 0, the plugin is disabled.
+     */
+    public static final HazelcastProperty PERIOD_SECONDS
+            = new HazelcastProperty("hazelcast.performance.monitor.overloaded.connections.period.seconds", 0, SECONDS);
+
+    /**
+     * The minimum number of packets in the connection before it is considered to be overloaded.
+     */
+    public static final HazelcastProperty THRESHOLD
+            = new HazelcastProperty("hazelcast.performance.monitor.overloaded.connections.threshold", 10000);
+
+    /**
+     * The number of samples to take from a single overloaded connection. Increasing the number of packes gives
+     * more accurracy of the content, but it will come at greater price.
+     */
+    public static final HazelcastProperty SAMPLES
+            = new HazelcastProperty("hazelcast.performance.monitor.overloaded.connections.samples", 1000);
+
     private static final Queue<OutboundFrame> EMPTY_QUEUE = new LinkedList<OutboundFrame>();
 
     private final SerializationService serializationService;
@@ -72,9 +100,9 @@ public OverloadedConnectionsPlugin(NodeEngineImpl nodeEngine) {
         this.defaultFormat.setMinimumFractionDigits(3);
 
         GroupProperties props = nodeEngine.getGroupProperties();
-        this.periodMillis = props.getMillis(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_PERIOD_SECONDS);
-        this.threshold = props.getInteger(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_THRESHOLD);
-        this.samples = props.getInteger(PERFORMANCE_MONITOR_OVERLOADED_CONNECTIONS_SAMPLES);
+        this.periodMillis = props.getMillis(PERIOD_SECONDS);
+        this.threshold = props.getInteger(THRESHOLD);
+        this.samples = props.getInteger(SAMPLES);
     }
 
     @Override