[serve] Fix serve agent 501 response (ray-project#39440)

TL;DR: the optimization added in ray-project#38836, which returns 501 if Serve dependencies aren't installed, doesn't work. Instead of returning a 501 response like it should, the serve agent returns a mysterious `TypeError: object Response can't be used in 'await' expression`

The decorator `gracefully_handle_missing_serve_dependencies` doesn't return an async function, and the second decorator `init_ray_and_catch_exceptions` runs `await f`, `f` being the result of the first decorator.
- Normally this works fine since `f` returns a coroutine (the original async function)
- But when serve dependencies are not installed and `f` returns a 501 Response, await on the 501 Response fails.

The tests `test_dashboard_does_not_depend_on_serve` and `test_agent_does_not_depend_on_serve` didn't catch this because it includes the assert statement inside a try-except block, so the assert statement was actually failing but it was being ignored by the except block 🤦

Fixed the test and also verified manually this time that the optimization works.

Author: zcin

dashboard/modules/serve/serve_agent.py

@@ -23,7 +23,7 @@
 
 def gracefully_handle_missing_serve_dependencies(func):
     @wraps(func)
-    def check(self, *args, **kwargs):
+    async def check(self, *args, **kwargs):
         try:
             from ray import serve  # noqa: F401
         except ImportError:
@@ -34,7 +34,7 @@ def check(self, *args, **kwargs):
                     '"ray[serve]"`.'
                 ),
             )
-        return func(self, *args, **kwargs)
+        return await func(self, *args, **kwargs)
 
     return check
 

dashboard/tests/test_dashboard.py

@@ -925,11 +925,11 @@ def test_dashboard_does_not_depend_on_serve():
     # Check that Serve-dependent features fail
     try:
         response = requests.get(f"http://{agent_url}/api/serve/deployments/")
+        print(f"response status code: {response.status_code}, expected: 501")
         assert response.status_code == 501
-    except Exception as e:
+    except requests.ConnectionError as e:
         # Fail to connect to service is fine.
         print(e)
-        assert True
 
 
 @pytest.mark.skipif(
@@ -963,11 +963,11 @@ def test_agent_does_not_depend_on_serve(shutdown_only):
     # Check that Serve-dependent features fail
     try:
         response = requests.get(f"http://{agent_url}/api/serve/deployments/")
+        print(f"response status code: {response.status_code}, expected: 501")
         assert response.status_code == 501
-    except Exception as e:
+    except requests.ConnectionError as e:
         # Fail to connect to service is fine.
         print(e)
-        assert True
 
     # The agent should be dead if raylet exits.
     raylet_proc.kill()