Clean up node start order (#136966)

Some service starts were added after http start, but these should all be
before starting http. In general nothing should be added after http
start. This commit moves the offending service starts earlier during
startup, and adds a loud comment to deter future such additions.

Author: rjernst

server/src/main/java/org/elasticsearch/node/Node.java

@@ -286,6 +286,10 @@ public Node start() throws NodeValidationException {
         injector.getInstance(RepositoriesService.class).start();
         injector.getInstance(SearchService.class).start();
         injector.getInstance(FsHealthService.class).start();
+        injector.getInstance(NodeMetrics.class).start();
+        injector.getInstance(IndicesMetrics.class).start();
+        injector.getInstance(HealthPeriodicLogger.class).start();
+        injector.getInstance(SamplingService.class).start();
         nodeService.getMonitorService().start();
 
         final ClusterService clusterService = injector.getInstance(ClusterService.class);
@@ -405,6 +409,8 @@ public void onTimeout(TimeValue timeout) {
             }
         }
 
+        // ------- DO NOT ADD NEW START CALLS BELOW HERE -------
+
         injector.getInstance(HttpServerTransport.class).start();
         if (ReadinessService.enabled(environment)) {
             injector.getInstance(ReadinessService.class).start();
@@ -425,12 +431,6 @@ public void onTimeout(TimeValue timeout) {
                 writePortsFile("remote_cluster", transport.boundRemoteAccessAddress());
             }
         }
-
-        injector.getInstance(NodeMetrics.class).start();
-        injector.getInstance(IndicesMetrics.class).start();
-        injector.getInstance(HealthPeriodicLogger.class).start();
-        injector.getInstance(SamplingService.class).start();
-
         logger.info("started {}", transportService.getLocalNode());
 
         pluginsService.filterPlugins(ClusterPlugin.class).forEach(ClusterPlugin::onNodeStarted);