Merge branch 'master' into functype-detection

Author: vEpiphyte

.circleci/config.yml

@@ -59,18 +59,10 @@ commands:
           command: |
             . venv/bin/activate
             mkdir test-reports
-            circleci tests glob synapse/tests/test_*.py synapse/vendor/**/test_*.py | circleci tests run --split-by=timings --command "xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml -o junit_family=xunit1 ${COVERAGE_ARGS}"
-
-  check_runner_has_work:
-    description: Check that this runner has work to do
-    steps:
-      - run:
-          name: Check workload
-          command: |
-            mkdir -p ./tmp && \
-            >./tmp/tests.txt && \
-            circleci tests glob synapse/tests/test_*.py synapse/vendor/**/test_*.py | circleci tests run --split-by=timings --command ">./tmp/tests.txt xargs echo"
-            [ -s tmp/tests.txt ] || circleci-agent step halt #if there are no tests, terminate execution after this step
+            circleci tests glob synapse/tests/test_*.py synapse/vendor/**/test_*.py | \
+              circleci tests run \
+                --timings-type=name \
+                --command="xargs python3 -m pytest -n 8 --dist worksteal -v -rs --durations 6 -p no:logging --junitxml=test-reports/junit.xml -o junit_family=xunit1 ${COVERAGE_ARGS}"
 
   test_steps_doc:
     description: "Documentation test steps"
@@ -121,7 +113,15 @@ commands:
     steps:
       - checkout
 
-      - check_runner_has_work
+      - run:
+          # Run this first so we fail on syntax errors before installing a bunch
+          # of stuff and doing a bunch of work. It's easier now that we're only
+          # a single runner using xdist.
+          name: syntax
+          command: |
+            pip install "pycodestyle>=2.10.0,<3.0.0"
+            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle synapse; fi;
+            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle scripts; fi;
 
       - run:
           name: checkout regression repo
@@ -144,15 +144,8 @@ commands:
             - ./venv
           key: v5-venv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }}-{{ checksum "/tmp/python.version" }}
 
-      - run:
-          name: syntax
-          command: |
-            . venv/bin/activate
-            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle synapse; fi;
-            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle scripts; fi;
-
-
       - do_test_execution
+
       - do_report_coverage
 
       - store_test_results:
@@ -439,7 +432,7 @@ commands:
 jobs:
 
   python311:
-    parallelism: 8
+    resource_class: xlarge
     docker:
       - image: cimg/python:3.11
         environment:
@@ -456,7 +449,7 @@ jobs:
       - test_steps_python
 
   python311_replay:
-    parallelism: 6
+    resource_class: xlarge
     docker:
       - image: cimg/python:3.11
         environment:
@@ -473,7 +466,6 @@ jobs:
       - test_steps_python
 
   doctests:
-    parallelism: 1
     docker:
       - image: cimg/python:3.11
         environment:

conftest.py

@@ -0,0 +1,28 @@
+import os
+import sys
+import warnings
+
+import synapse.common as s_common
+
+THROW = False
+
+def audithook(event, args):
+    if event == 'socket.bind':
+        _, addr = args
+        if isinstance(addr, tuple) and (port := addr[1]) != 0:
+
+            testname = os.environ.get('PYTEST_CURRENT_TEST', '<unknown>').split(' ')[0]
+
+            mesg = f'Synapse tests should not bind to fixed ports: {testname=} {port=}'
+            warnings.warn(mesg)
+
+            if THROW:
+                raise RuntimeError(mesg)
+
+def pytest_sessionstart(session):
+    if s_common.envbool('SYNDEV_AUDIT_PORT_BINDS'):
+        sys.addaudithook(audithook)
+
+    if s_common.envbool('SYNDEV_AUDIT_PORT_BINDS_RAISE'):
+        global THROW
+        THROW = True

synapse/tests/test_lib_aha.py

@@ -59,7 +59,7 @@ async def test_lib_aha_clone(self):
                     self.len(ahacount, await proxy0.getAhaUrls())
                     self.len(ahacount, await proxy0.getAhaServers())
 
-                    purl = await proxy0.addAhaClone(zoinks)
+                    purl = await proxy0.addAhaClone(zoinks, port=0)
 
                 conf1 = {'clone': purl}
                 async with self.getTestAha(conf=conf1, dirn=dir1) as aha1:
@@ -574,7 +574,7 @@ async def test_lib_aha_provision(self):
                 }
                 s_common.yamlsave(axonconf, axonpath, 'cell.yaml')
 
-                argv = (axonpath, '--auth-passwd', 'rootbeer')
+                argv = (axonpath, '--auth-passwd', 'rootbeer', '--https', '0')
                 async with await s_axon.Axon.initFromArgv(argv) as axon:
 
                     # opts were copied through successfully
@@ -1161,7 +1161,8 @@ async def test_aha_provision_longname(self):
                     aconf = {
                         'aha:name': 'aha',
                         'aha:network': networkname,
-                        'provision:listen': f'ssl://aha.{networkname}:0'
+                        'dmon:listen': f'ssl://aha.{networkname}:0',
+                        'provision:listen': f'ssl://aha.{networkname}:0',
                     }
                     name = aconf.get('aha:name')
                     netw = aconf.get('aha:network')
@@ -1300,11 +1301,17 @@ async def test_aha_provision_listen_dns_name(self):
         conf = {
             'aha:network': 'synapse',
             'dns:name': 'here.loop.vertex.link',
+            'dmon:listen': 'ssl://0.0.0.0:0?hostname=here.loop.vertex.link&ca=synapse',
         }
-        mesg = 'provision listening: ssl://0.0.0.0:27272?hostname=here.loop.vertex.link'
-        with self.getAsyncLoggerStream('synapse.lib.aha', mesg) as stream:
+
+        orig = s_aha.AhaCell._getProvListen
+        def _getProvListen(_self):
+            ret = orig(_self)
+            self.eq(ret, 'ssl://0.0.0.0:27272?hostname=here.loop.vertex.link')
+            return 'ssl://0.0.0.0:0?hostname=here.loop.vertex.link'
+
+        with mock.patch('synapse.lib.aha.AhaCell._getProvListen', _getProvListen):
             async with self.getTestCell(s_aha.AhaCell, conf=conf) as aha:
-                self.true(await stream.wait(timeout=6))
                 # And the URL works with our listener :)
                 provurl = await aha.addAhaUserEnroll('bob.grey')
                 async with await s_telepath.openurl(provurl) as prox:

synapse/tests/test_tools_healthcheck.py

@@ -45,7 +45,7 @@ async def sleep(*args, **kwargs):
                 await asyncio.sleep(0.6)
             core.addHealthFunc(sleep)
             outp.clear()
-            retn = await s_t_healthcheck.main(['-c', curl, '-t', '0.2'], outp)
+            retn = await s_t_healthcheck.main(['-c', curl, '-t', '0.4'], outp)
             self.eq(retn, 1)
             resp = json.loads(str(outp))
             self.eq(resp.get('components')[0].get('name'), 'error')
@@ -58,7 +58,7 @@ async def sleep(*args, **kwargs):
             _, port = await core.dmon.listen('tcp://127.0.0.1:0')
             root = await core.auth.getUserByName('root')
             await root.setPasswd('secret')
-            retn = await s_t_healthcheck.main(['-c', f'tcp://root:[email protected]:{port}/cortex', '-t', '0.2'], outp)
+            retn = await s_t_healthcheck.main(['-c', f'tcp://root:[email protected]:{port}/cortex', '-t', '0.4'], outp)
             self.eq(retn, 1)
             resp = json.loads(str(outp))
             self.eq(resp.get('components')[0].get('name'), 'error')
@@ -70,7 +70,7 @@ async def sleep(*args, **kwargs):
 
             logger.info('Checking without perms')
             outp.clear()
-            retn = await s_t_healthcheck.main(['-c', f'tcp://visi:[email protected]:{port}/cortex', '-t', '0.2'], outp)
+            retn = await s_t_healthcheck.main(['-c', f'tcp://visi:[email protected]:{port}/cortex', '-t', '0.4'], outp)
             self.eq(retn, 1)
             resp = json.loads(str(outp))
             self.eq(resp.get('components')[0].get('name'), 'error')
@@ -83,7 +83,7 @@ async def sleep(*args, **kwargs):
             await core.fini()
             await asyncio.sleep(0)
             outp.clear()
-            retn = await s_t_healthcheck.main(['-c', curl, '-t', '0.2'], outp)
+            retn = await s_t_healthcheck.main(['-c', curl, '-t', '0.4'], outp)
             self.eq(retn, 1)
             resp = json.loads(str(outp))
             self.eq(resp.get('components')[0].get('name'), 'error')