chore(testinfra): check for connectivity

.github/workflows/testinfra.yml

@@ -127,7 +127,7 @@ jobs:
         run: |
           # TODO: use poetry for pkg mgmt
           pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
-          pytest -vv testinfra/test_ami.py
+          pytest -vv -s testinfra/test_ami.py
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}

testinfra/test_ami.py

@@ -5,10 +5,10 @@
 import os
 import pytest
 import requests
+import socket
 import testinfra
 from ec2instanceconnectcli.EC2InstanceConnectLogger import EC2InstanceConnectLogger
 from ec2instanceconnectcli.EC2InstanceConnectKey import EC2InstanceConnectKey
-from paramiko.ssh_exception import NoValidConnectionsError
 from time import sleep
 
 RUN_ID = os.environ.get("GITHUB_RUN_ID", "unknown-ci-run")
@@ -243,7 +243,18 @@ def gzip_then_base64_encode(s: str) -> str:
     assert response["Success"]
 
     # instance doesn't have public ip yet
-    instance.reload()
+    while not instance.public_ip_address:
+        logger.warning("waiting for ip to be available")
+        sleep(5)
+        instance.reload()
+
+    while True:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if sock.connect_ex((instance.public_ip_address, 22)) == 0:
+            break
+        else:
+            logger.warning("waiting for ssh to be available")
+            sleep(10)
 
     host = testinfra.get_host(
         # paramiko is an ssh backend
@@ -252,51 +263,46 @@ def gzip_then_base64_encode(s: str) -> str:
     )
 
     def is_healthy(host) -> bool:
-        try:
-            cmd = host.run("pg_isready -U postgres")
-            if cmd.failed is True:
-                logger.warn("pg not ready")
-                return False
+        cmd = host.run("pg_isready -U postgres")
+        if cmd.failed is True:
+            logger.warning("pg not ready")
+            return False
 
-            cmd = host.run(f"curl -sf -k https://localhost:8085/health -H 'apikey: {supabase_admin_key}'")
-            if cmd.failed is True:
-                logger.warn("adminapi not ready")
-                return False
+        cmd = host.run(f"curl -sf -k https://localhost:8085/health -H 'apikey: {supabase_admin_key}'")
+        if cmd.failed is True:
+            logger.warning("adminapi not ready")
+            return False
 
-            cmd = host.run("curl -sf http://localhost:3001/ready")
-            if cmd.failed is True:
-                logger.warn("postgrest not ready")
-                return False
+        cmd = host.run("curl -sf http://localhost:3001/ready")
+        if cmd.failed is True:
+            logger.warning("postgrest not ready")
+            return False
 
-            cmd = host.run("curl -sf http://localhost:8081/health")
-            if cmd.failed is True:
-                logger.warn("gotrue not ready")
-                return False
+        cmd = host.run("curl -sf http://localhost:8081/health")
+        if cmd.failed is True:
+            logger.warning("gotrue not ready")
+            return False
 
-            cmd = host.run("sudo kong health")
-            if cmd.failed is True:
-                logger.warn("kong not ready")
-                return False
+        cmd = host.run("sudo kong health")
+        if cmd.failed is True:
+            logger.warning("kong not ready")
+            return False
 
-            cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'")
-            if cmd.failed is True:
-                logger.warn("pgbouncer not ready")
-                return False
+        cmd = host.run("printf \\\\0 > '/dev/tcp/localhost/6543'")
+        if cmd.failed is True:
+            logger.warning("pgbouncer not ready")
+            return False
 
-            cmd = host.run("sudo fail2ban-client status")
-            if cmd.failed is True:
-                logger.warn("fail2ban not ready")
-                return False
-        except NoValidConnectionsError:
-            logger.warn("unable to connect via ssh")
+        cmd = host.run("sudo fail2ban-client status")
+        if cmd.failed is True:
+            logger.warning("fail2ban not ready")
             return False
 
         return True
 
     while True:
         if is_healthy(host):
             break
-        print("waiting until healthy")
         sleep(1)
 
     # return a testinfra connection to the instance