testuite: Python3 support

ASTERISK-26826

Change-Id: Id22c8f7a89e2d00fca760970af63f3954c049bf3

Author: mbradeen

.gitignore

@@ -9,4 +9,5 @@ crash-mail-config.yaml
 /fastagi
 /run-local.conf
 /.valgrindrc
+/.venv
 __pycache__/

addons/Makefile

@@ -23,18 +23,20 @@ distclean: _clean
 	rm -rf starpy/install.record
 
 install:
-	(cd starpy; python setup.py install --prefix=~/.local --record install.record)
+	(cd starpy; python3 setup.py install --prefix=~/.local --record install.record)
 
 uninstall:
 	rm -rf $$(cat starpy/install.record)
 
 update:
 	@if [ -d starpy ]; then \
 		cd starpy; \
-		if [ `git config --get remote.origin.url` = git://github.com/asterisk-org/starpy.git ]; then \
-			git remote set-url origin git://github.com/asterisk/starpy.git git://github.com/asterisk-org/starpy.git; \
+		if [ `git config --get remote.origin.url` = https://github.com/asterisk-org/starpy.git ]; then \
+			git remote set-url origin https://github.com/asterisk/starpy.git https://github.com/asterisk-org/starpy.git; \
 		fi; \
+		git checkout 1.1; \
 		git pull; \
 	else \
-		git clone git://github.com/asterisk/starpy.git; \
+		git clone https://github.com/asterisk/starpy.git; \
+		git checkout 1.1; \
 	fi

contrib/scripts/cel-to-yaml.py

@@ -48,7 +48,7 @@ def main(argv=None):
         result.append(blob)
     raw_yaml = yaml.dump(result)
     raw_yaml = raw_yaml.replace(',', '\n').replace('}', '').replace('{', '')
-    print raw_yaml
+    print(raw_yaml)
 
     return 0
 

contrib/scripts/install_prereq

@@ -15,24 +15,20 @@ usage() {
 # Basic build system:
 PACKAGES_DEBIAN="subversion git build-essential automake"
 # Python tools
-PACKAGES_DEBIAN="$PACKAGES_DEBIAN python python-pip python-setuptools python-dev cython"
+PACKAGES_DEBIAN="$PACKAGES_DEBIAN python3 python3-pip python3-setuptools python3-dev"
 # Testsuite: basic requirements:
-PACKAGES_DEBIAN="$PACKAGES_DEBIAN python-yaml python-twisted-core python-lxml liblua5.1-0-dev lua5.1 gdb"
+PACKAGES_DEBIAN="$PACKAGES_DEBIAN liblua5.1-0-dev lua5.1 gdb"
 # Sipp requirements
 PACKAGES_DEBIAN="$PACKAGES_DEBIAN libpcap-dev libssl-dev libsctp-dev libncurses5-dev"
-# pjsua requirements
-#PACKAGES_DEBIAN="$PACKAGES_DEBIAN python-dev"
 
 # Basic build system:
 PACKAGES_RH="subversion git gcc gcc-c++ patch"
 # Python tools
-PACKAGES_RH="$PACKAGES_RH python python-pip python-setuptools python-devel Cython"
+PACKAGES_RH="$PACKAGES_RH python3 python3-pip python3-setuptools python3-devel"
 # Testsuite: basic requirements:
-PACKAGES_RH="$PACKAGES_RH PyYAML python-twisted-core python-lxml lua-devel gdb"
+PACKAGES_RH="$PACKAGES_RH lua-devel gdb"
 # Sipp requirements
 PACKAGES_RH="$PACKAGES_RH libpcap-devel openssl-devel lksctp-tools-devel"
-# pjsua requirements
-#PACKAGES_RH="$PACKAGES_RH python-devel"
 
 LIBDIR=/usr/lib$(getconf LONG_BIT | fgrep 64)
 [ ! -d $LIBDIR ] && LIBDIR=/usr/lib
@@ -145,38 +141,13 @@ else
 	exit 1
 fi
 
-install_starpy() {
-	$testcmd cd addons
-	$testcmd make update
-	$testcmd cd starpy
-	$testcmd sudo python setup.py install
-	$testcmd cd ../..
-}
-
 install_asttest() {
 	$testcmd cd asttest
 	$testcmd make
 	$testcmd sudo make install
 	$testcmd cd ..
 }
 
-# Build and install yappcap
-install_yappcap() {
-	SAVE_DIR=`pwd`
-	$testcmd cd /tmp
-
-	PROJECT_DIR=yappcap
-
-	$testcmd sudo rm -rf $PROJECT_DIR
-	$testcmd git clone https://github.com/otherwiseguy/yappcap $PROJECT_DIR
-	$testcmd cd $PROJECT_DIR
-
-	$testcmd make
-	$testcmd sudo make install
-
-	$testcmd cd $SAVE_DIR
-}
-
 # Build and install sipp
 install_sipp() {
 	SAVE_DIR=`pwd`
@@ -223,7 +194,6 @@ install_pjsua() {
 
 	# The testsuite only cares about pjsua from what we have built.
 	$testcmd sudo cp -v pjsip-apps/bin/pjsua-`uname -m`-*-linux-gnu $MY_BIN/pjsua
-	$testcmd sudo make -C pjsip-apps/src/python install
 
 	$testcmd cd $SAVE_DIR
 }
@@ -232,24 +202,9 @@ install_testsuite_dependencies()
 {
 	$install_packages
 
-	if [ "x`find $LIBDIR/python* $LIBLOCALDIR/python* -name 'starpy*'`" = "x" ] ; then
-		install_starpy
-	fi
-
-	if [ "x`find $LIBDIR/python* $LIBLOCALDIR/python* -name yappcap.so`" = "x" ] ; then
-		install_yappcap
-	fi
-
 	which asttest > /dev/null || install_asttest
 	which sipp > /dev/null || install_sipp
 	which pjsua > /dev/null || install_pjsua
-
-	$testcmd sudo pip install construct
-	$testcmd sudo pip install requests
-
-	# The following is needed for ARI tests and may not install
-	# on older distributions.
-	$testcmd sudo pip install autobahn
 }
 
 if in_test_mode; then
@@ -266,7 +221,7 @@ if ! in_test_mode; then
 	echo "## $1 completed successfully"
 	echo "#############################################"
 	echo ""
-	echo "Testsuite requires 2.6.5 <= python version < 3"
+	echo "Testsuite requires python version >= 3.6.8"
 	echo "Your installed python is"
 	python --version
 fi

contrib/scripts/rlmi_demo.py

@@ -23,46 +23,46 @@
 list_elem = rlmi.CreateFromDocument(xml)
 
 # The outermost XML element in an RLMI document is the 'list' element.
-print 'list'
+print('list')
 # pyxb performs type-checking and type conversion of elements that it comes
 # across in XML documents. In this case, the list element's version attribute is
 # an integer, and the fullState attribute is a boolean. This is why the str()
 # function is necessary in order to convert to string for the concatenation.
-print '\t' + str(list_elem.version)
-print '\t' + str(list_elem.fullState)
-print '\t' + list_elem.uri
+print('\t' + str(list_elem.version))
+print('\t' + str(list_elem.fullState))
+print('\t' + list_elem.uri)
 # A list element may have zero or more name elements in it. This is a
 # user-visible name for the list. The main reason why more than one name may
 # exist for a list would be because it is expressed in multiple languages.
 # Asterisk RLMI lists will have only a single name.
 for name in list_elem.name:
-    print '\tname'
+    print('\tname')
     # Parsed XML documents have their attributes accessed as members on an
     # object.
-    print '\t\t' + name.lang
+    print('\t\t' + name.lang)
     # The content of XML elements is accessed through the value() member.
-    print '\t\t' + name.value()
+    print('\t\t' + name.value())
 # A list element may have zero or more resource elements in it. The resources
 # represent the resources that make up the list (duh).
 for resource in list_elem.resource:
-    print '\tresource'
-    print '\t\t' + resource.uri
+    print('\tresource')
+    print('\t\t' + resource.uri)
     # Resources may have names associated with them, just like the list does.
     # Asterisk will use the name of the list from the configuration file here.
     for name in resource.name:
-        print '\t\tname'
-        print '\t\t\t' + name.value()
+        print('\t\tname')()
+        print('\t\t\t' + name.value())
     # Resources may have zero or more instance elements on them. The reason that
     # more than one instance element may exist for a resource is that a resource
     # may correspond to a single subscription that forks, resulting in multiple
     # instances of the resource being represented. In Asterisk's case, there
     # will be a single instance per resource.
     for instance in resource.instance:
-        print '\t\tinstance'
-        print '\t\t\t' + instance.id
-        print '\t\t\t' + instance.state
+        print('\t\tinstance')
+        print('\t\t\t' + instance.id)
+        print('\t\t\t' + instance.state)
         # If an instance has a cid, it indicates that there is a body element
         # in the multipart body that corresponds to the instance. The cid
         # corresponds to the Content-ID header in that body part.
         if instance.cid:
-            print '\t\t\t' + instance.cid
+            print('\t\t\t' + instance.cid)

extras.txt

@@ -0,0 +1,2 @@
+https://github.com/asterisk/starpy/archive/refs/heads/1.1.zip
+https://github.com/asterisk/yappcap/archive/refs/heads/python3.zip

lib/python/asterisk/astconfigparser.py

@@ -119,7 +119,7 @@ def keys(self, self_only=False):
             for key in d.keys():
                 if key not in res:
                     res.append(key)
-        return res
+        return list(res.keys())
 
     def add_defaults(self, defaults):
         """

lib/python/asterisk/astdicts.py

@@ -271,11 +271,11 @@ def __init__(self, *args, **kwds):
 
     def __setitem__(self, key, val, i=None):
         if key not in self:
-#            print "__setitem__ key = ", key, " val = ", val
+#            print("__setitem__ key = ", key, " val = ", val)
             OrderedDict.__setitem__(
                 self, key, val if isinstance(val, list) else [val])
             return
-#        print "inserting key = ", key, " val = ", val
+#        print("inserting key = ", key, " val = ", val)
         vals = self[key]
         if i is None:
             i = len(vals)

lib/python/asterisk/asterisk.py

@@ -243,7 +243,7 @@ def connectionMade(self):
 
     def errReceived(self, data):
         """Override of ProcessProtocol.errReceived"""
-        LOGGER.warn("Asterisk %s received error: %s" % (self._host, data))
+        LOGGER.warn("Asterisk %s received error: %s" % (self._host, data.decode('utf-8', 'ignore')))
 
     def processEnded(self, reason):
         """Override of ProcessProtocol.processEnded"""
@@ -464,8 +464,12 @@ def __start_asterisk_callback(cmd):
             self.process = reactor.spawnProcess(self.protocol,
                                                 cmd[0],
                                                 cmd, env=os.environ)
-            # Begin the wait fully booted cycle
-            reactor.callLater(1, __execute_wait_fully_booted)
+            # This was a one second delay, now two.  This is to allow
+            # asterisk sufficient time to actually start and create the ctl
+            # file. If we try to send the fully booted command before this
+            # happens we wait and try again, but this results in an unhandled
+            # error in twisted after the command succeeds.
+            reactor.callLater(2, __execute_wait_fully_booted)
 
         def __execute_wait_fully_booted():
             """Send the CLI command waitfullybooted"""
@@ -496,8 +500,9 @@ def __wait_fully_booted_error(cli_command):
                 LOGGER.error(msg)
                 self._start_deferred.errback(Exception(msg))
             else:
-                msg = "Asterisk core waitfullybooted failed, attempting again"
-                LOGGER.debug(msg)
+                LOGGER.debug("Asterisk core waitfullybooted failed " +
+                             "with output '%s', attempting again..." %
+                             cli_command.value.err.decode('utf-8'))
                 reactor.callLater(1, __execute_wait_fully_booted)
             return cli_command