[LSC] Avoid relaunching the daemon if the socket is probably bound (#1489)

Author: JakeSiFive

Diff for: src/job_cache/job_cache.cpp

@@ -134,7 +134,9 @@ static bool daemonize(std::string dir) {
   return true;
 }
 
-wcl::optional<wcl::unique_fd> try_connect(std::string dir) {
+enum class TryConnectError { Generic, AddrBound };
+
+wcl::result<wcl::unique_fd, TryConnectError> try_connect(std::string dir) {
   wcl::unique_fd socket_fd;
   {
     int local_socket_fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
@@ -151,7 +153,7 @@ wcl::optional<wcl::unique_fd> try_connect(std::string dir) {
   auto fd = wcl::unique_fd::open(key_path.c_str(), O_RDONLY);
   if (!fd) {
     wcl::log::info("open(%s): %s", key_path.c_str(), strerror(fd.error()))();
-    return {};
+    return wcl::result_error<wcl::unique_fd>(TryConnectError::Generic);
   }
 
   // TODO: We should make this read more robust. It's mostly fine if
@@ -167,14 +169,20 @@ wcl::optional<wcl::unique_fd> try_connect(std::string dir) {
   addr.sun_family = AF_UNIX;
   addr.sun_path[0] = '\0';
 
+  // Among the failures that can occur here, we could get EAGAIN. We can't use
+  // epoll to avoid this however if we get EAGAIN it means that we don't have
+  // to launch another process because the address is already bound.
   wcl::log::info("key = %s, sizeof(key) = %lu", key, sizeof(key))();
   memcpy(addr.sun_path + 1, key, sizeof(key));
   if (connect(socket_fd.get(), reinterpret_cast<const sockaddr *>(&addr), sizeof(key)) == -1) {
+    if (errno == EAGAIN) {
+      return wcl::result_error<wcl::unique_fd>(TryConnectError::AddrBound);
+    }
     wcl::log::info("connect(%s): %s", key, strerror(errno))();
-    return {};
+    return wcl::result_error<wcl::unique_fd>(TryConnectError::Generic);
   }
 
-  return wcl::make_some<wcl::unique_fd>(std::move(socket_fd));
+  return wcl::result_value<TryConnectError>(std::move(socket_fd));
 }
 
 // Launch the job cache daemon
@@ -209,16 +217,27 @@ wcl::result<wcl::unique_fd, ConnectError> Cache::backoff_try_connect(int attempt
   wcl::xoshiro_256 rng(wcl::xoshiro_256::get_rng_seed());
   useconds_t backoff = 1000;
   wcl::unique_fd socket_fd;
+  bool addr_bound = false;
   for (int i = 0; i < attempts; i++) {
     // We normally connect in about 3 tries, sometimes 4 on fresh
     // connect so if we haven't connected at this point its a good
     // spot to start start trying.
-    if (i > 4) {
+    // Additionally if on the previous connection attempt we gained evidence
+    // that the connection is already bound, we'll avoid relaunching for at
+    // most 1 round. This halves the number of extranious daemon launches
+    // when the daemon is under load.
+    if (i > 4 && !addr_bound) {
       launch_daemon();
     }
 
+    addr_bound = false;
     auto fd_opt = try_connect(cache_dir);
     if (!fd_opt) {
+      // If we receive this specific error then we know that a daemon exists,
+      // its just busy
+      if (fd_opt.error() == TryConnectError::AddrBound) {
+        addr_bound = true;
+      }
       std::uniform_int_distribution<useconds_t> variance(0, backoff);
       usleep(backoff + variance(rng));
       backoff *= 2;