Report system resource metrics from background tasks

signalapp · Jun 14, 2023 · 13afdbd · 13afdbd
1 parent 9cfd88a
commit 13afdbd
Show file tree

Hide file tree

Showing 5 changed files with 98 additions and 58 deletions.
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java
@@ -46,7 +46,6 @@
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import javax.servlet.DispatcherType;
 import javax.servlet.FilterRegistration;
 import javax.servlet.ServletRegistration;
@@ -127,18 +126,8 @@
 import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper;
 import org.whispersystems.textsecuregcm.mappers.RegistrationServiceSenderExceptionMapper;
 import org.whispersystems.textsecuregcm.mappers.ServerRejectedExceptionMapper;
-import org.whispersystems.textsecuregcm.metrics.ApplicationShutdownMonitor;
-import org.whispersystems.textsecuregcm.metrics.BufferPoolGauges;
-import org.whispersystems.textsecuregcm.metrics.CpuUsageGauge;
-import org.whispersystems.textsecuregcm.metrics.FileDescriptorGauge;
-import org.whispersystems.textsecuregcm.metrics.FreeMemoryGauge;
-import org.whispersystems.textsecuregcm.metrics.GarbageCollectionGauges;
-import org.whispersystems.textsecuregcm.metrics.MaxFileDescriptorGauge;
 import org.whispersystems.textsecuregcm.metrics.MetricsApplicationEventListener;
 import org.whispersystems.textsecuregcm.metrics.MetricsUtil;
-import org.whispersystems.textsecuregcm.metrics.NetworkReceivedGauge;
-import org.whispersystems.textsecuregcm.metrics.NetworkSentGauge;
-import org.whispersystems.textsecuregcm.metrics.OperatingSystemMemoryGauge;
 import org.whispersystems.textsecuregcm.metrics.ReportedMessageMetricsListener;
 import org.whispersystems.textsecuregcm.metrics.TrafficSource;
 import org.whispersystems.textsecuregcm.providers.MultiRecipientMessageProvider;
@@ -783,7 +772,6 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
       webSocketEnvironment.jersey().register(controller);
     }
 
-
     WebSocketEnvironment<AuthenticatedAccount> provisioningEnvironment = new WebSocketEnvironment<>(environment,
         webSocketEnvironment.getRequestLog(), 60000);
     provisioningEnvironment.jersey().register(new WebsocketRefreshApplicationEventListener(accountsManager, clientPresenceManager));
@@ -817,21 +805,7 @@ public void run(WhisperServerConfiguration config, Environment environment) thro
 
     environment.healthChecks().register("cacheCluster", new RedisClusterHealthCheck(cacheCluster));
 
-    environment.lifecycle().manage(new ApplicationShutdownMonitor(Metrics.globalRegistry));
-
-    environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge(3, TimeUnit.SECONDS));
-    environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
-    environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
-    environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
-    environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
-    environment.metrics().register(name(MaxFileDescriptorGauge.class, "max_fd_count"), new MaxFileDescriptorGauge());
-    environment.metrics()
-        .register(name(OperatingSystemMemoryGauge.class, "buffers"), new OperatingSystemMemoryGauge("Buffers"));
-    environment.metrics()
-        .register(name(OperatingSystemMemoryGauge.class, "cached"), new OperatingSystemMemoryGauge("Cached"));
-
-    BufferPoolGauges.registerMetrics();
-    GarbageCollectionGauges.registerMetrics();
+    MetricsUtil.registerSystemResourceMetrics(environment);
   }
 
 

diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java
@@ -5,14 +5,19 @@
 
 package org.whispersystems.textsecuregcm.metrics;
 
+import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.SharedMetricRegistries;
+import io.dropwizard.lifecycle.JettyManaged;
+import io.dropwizard.lifecycle.setup.LifecycleEnvironment;
+import io.dropwizard.metrics.ScheduledReporterManager;
 import io.dropwizard.setup.Environment;
 import io.micrometer.core.instrument.Meter;
 import io.micrometer.core.instrument.Metrics;
 import io.micrometer.core.instrument.Tags;
 import io.micrometer.core.instrument.config.MeterFilter;
 import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
 import io.micrometer.datadog.DatadogMeterRegistry;
+import java.util.concurrent.TimeUnit;
 import org.whispersystems.textsecuregcm.WhisperServerConfiguration;
 import org.whispersystems.textsecuregcm.WhisperServerVersion;
 import org.whispersystems.textsecuregcm.util.Constants;
@@ -66,5 +71,60 @@ public DistributionStatisticConfig configure(final Meter.Id id, final Distributi
     }
 
     environment.lifecycle().manage(new MicrometerRegistryManager(Metrics.globalRegistry));
+    environment.lifecycle().manage(new ApplicationShutdownMonitor(Metrics.globalRegistry));
+  }
+
+  public static void registerSystemResourceMetrics(final Environment environment) {
+    environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge(3, TimeUnit.SECONDS));
+    environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
+    environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
+    environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
+    environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
+    environment.metrics().register(name(MaxFileDescriptorGauge.class, "max_fd_count"), new MaxFileDescriptorGauge());
+    environment.metrics()
+        .register(name(OperatingSystemMemoryGauge.class, "buffers"), new OperatingSystemMemoryGauge("Buffers"));
+    environment.metrics()
+        .register(name(OperatingSystemMemoryGauge.class, "cached"), new OperatingSystemMemoryGauge("Cached"));
+
+    BufferPoolGauges.registerMetrics();
+    GarbageCollectionGauges.registerMetrics();
+  }
+
+  /**
+   * For use in commands where {@link JettyManaged} doesn't apply
+   *
+   * @see io.dropwizard.metrics.MetricsFactory#configure(LifecycleEnvironment, MetricRegistry)
+   */
+  public static void startManagedReporters(Environment environment) {
+    environment.lifecycle().getManagedObjects().forEach(managedObject -> {
+      if (managedObject instanceof JettyManaged jettyManaged) {
+        if (jettyManaged.getManaged() instanceof ScheduledReporterManager scheduledReporterManager) {
+          try {
+            scheduledReporterManager.start();
+          } catch (final Exception e) {
+            throw new RuntimeException(e);
+          }
+        }
+      }
+    });
+  }
+
+  /**
+   * For use in commands where {@link JettyManaged} doesn't apply
+   *
+   * @see io.dropwizard.metrics.MetricsFactory#configure(LifecycleEnvironment, MetricRegistry)
+   */
+  public static void stopManagedReporters(final Environment environment) {
+    environment.lifecycle().getManagedObjects().forEach(lifeCycle -> {
+      if (lifeCycle instanceof JettyManaged jettyManaged) {
+        if (jettyManaged.getManaged() instanceof ScheduledReporterManager scheduledReporterManager) {
+          try {
+            scheduledReporterManager.stop();
+          } catch (final Exception e) {
+            throw new RuntimeException(e);
+          }
+        }
+      }
+    });
   }
 }
diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/workers/CrawlAccountsCommand.java b/service/src/main/java/org/whispersystems/textsecuregcm/workers/CrawlAccountsCommand.java
@@ -97,44 +97,46 @@ protected void run(final Environment environment, final Namespace namespace,
             DynamicConfiguration.class);
 
     dynamicConfigurationManager.start();
+    MetricsUtil.registerSystemResourceMetrics(environment);
+
+    final AccountDatabaseCrawler crawler = switch ((CrawlType) namespace.get(CRAWL_TYPE)) {
+      case GENERAL_PURPOSE -> {
+        final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
+            cacheCluster,
+            AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
+
+        yield new AccountDatabaseCrawler("General-purpose account crawler",
+            accountsManager,
+            accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
+            configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
+            dynamicConfigurationManager
+        );
+      }
+      case ACCOUNT_CLEANER -> {
+        final ExecutorService accountDeletionExecutor = environment.lifecycle()
+            .executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
+
+        final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
+            cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
+
+        yield new AccountDatabaseCrawler("Account cleaner crawler",
+            accountsManager,
+            accountDatabaseCrawlerCache,
+            List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
+            configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
+            dynamicConfigurationManager
+        );
+      }
+    };
 
-    final AccountDatabaseCrawler crawler =
-
-        switch ((CrawlType) namespace.get(CRAWL_TYPE)) {
-          case GENERAL_PURPOSE -> {
-            final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
-                cacheCluster,
-                AccountDatabaseCrawlerCache.GENERAL_PURPOSE_PREFIX);
-
-            yield new AccountDatabaseCrawler("General-purpose account crawler",
-                accountsManager,
-                accountDatabaseCrawlerCache, accountDatabaseCrawlerListeners,
-                configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
-                dynamicConfigurationManager
-            );
-          }
-          case ACCOUNT_CLEANER -> {
-            final ExecutorService accountDeletionExecutor = environment.lifecycle()
-                .executorService(name(getClass(), "accountCleaner-%d")).maxThreads(16).minThreads(16).build();
-
-            final AccountDatabaseCrawlerCache accountDatabaseCrawlerCache = new AccountDatabaseCrawlerCache(
-                cacheCluster, AccountDatabaseCrawlerCache.ACCOUNT_CLEANER_PREFIX);
-
-            yield new AccountDatabaseCrawler("Account cleaner crawler",
-                accountsManager,
-                accountDatabaseCrawlerCache,
-                List.of(new AccountCleaner(accountsManager, accountDeletionExecutor)),
-                configuration.getAccountDatabaseCrawlerConfiguration().getChunkSize(),
-                dynamicConfigurationManager
-            );
-          }
-        };
+    MetricsUtil.startManagedReporters(environment);
 
     try {
       crawler.crawlAllAccounts();
     } catch (final Exception e) {
       LoggerFactory.getLogger(CrawlAccountsCommand.class).error("Error crawling accounts", e);
     }
 
+    MetricsUtil.stopManagedReporters(environment);
   }
 }
diff --git a/...rc/main/java/org/whispersystems/textsecuregcm/workers/MessagePersisterServiceCommand.java b/...rc/main/java/org/whispersystems/textsecuregcm/workers/MessagePersisterServiceCommand.java
@@ -70,6 +70,8 @@ protected void run(Environment environment, Namespace namespace, WhisperServerCo
     environment.lifecycle().manage(deps.messagesCache());
     environment.lifecycle().manage(messagePersister);
 
+    MetricsUtil.registerSystemResourceMetrics(environment);
+
     super.run(environment, namespace, configuration);
   }
 

diff --git a/...hispersystems/textsecuregcm/workers/ScheduledApnPushNotificationSenderServiceCommand.java b/...hispersystems/textsecuregcm/workers/ScheduledApnPushNotificationSenderServiceCommand.java
@@ -79,6 +79,8 @@ protected void run(Environment environment, Namespace namespace, WhisperServerCo
     environment.lifecycle().manage(apnSender);
     environment.lifecycle().manage(apnPushNotificationScheduler);
 
+    MetricsUtil.registerSystemResourceMetrics(environment);
+
     super.run(environment, namespace, configuration);
   }