Begin work on python3 compatability.

* Use explicit relative import syntax.
* Correct syntax for print and except.
* Fix incorrect usage of IOError in astcdr.py - IOError is not a tuple.
* Use items, values instead of iteritems, itervalues.
* Use OrderedDict from collections if available (python 2.7+).
* Do not unpack tuples from argument list, save tuple to variable then
  unpack within code.
* Use exception handler to import Python3 version of urlencode if the
  Python2 version fails to import.
* Remove unused import unittest from opensslversion.
* Move executable tests to separate files to resolve issues with
  relative imports:
  - buildoptions
  - cdr
  - cel
  - channel_test_condition
  - config
  - lock_test_condition
  - sip_dialog_test_condition (disabled)
  - sippversion
* Modify utils_socket test to use new harness_shared module.
* Remove unused executable flag and shebang from phones.py.
* Update self_test script to run tests from new location.
* Remove ability to use config.py to display a config file.
* Allow PYTHON environmental variable to tell run-local to use a
  specific python binary.
* Execute test_runner.py using python -m syntax.  This avoids conflicts
  between needing to use relative imports for modules but not being
  allowed to use them in executable scripts.
* Encode self.realbase for use with md5 function.
* Remove executable flag and shebang from test_runner.py.  Remove
  redundant appending of sys.path.
* Modify output for running test and status to print the name of the
  test instead of the command used to run the test.
* Use key= sorter argument to compare sizes of tmp filesystems.  This
  may cause us to use /var/tmp instead of /tmp even if /var/tmp has only
  a few kilobytes more space but python3 does not support cmp= argument
  to sorter.
* Decode binary to utf-8
  - CLI output
  - runtest.py subprocess output
  - XML printed to stdout
  - SyncAMI data
  - SIPp runtime and version output
* Update test_runner.load_and_parse_module to check for the module in
  lib/python/asterisk, make required adjustments if found.
* Update __strip_illegal_xml_chars to work with unicode version of
  str.translate which does not support second argument.
* Update import statements from all *.py modules within tests/ to use
  absolute names for asterisk modules.  This is required for python2 to
  continue functioning with the new way that test_runner must be executed.
  No attempt has been made to fix python3 compatibility issues in these
  modules.

Some tests may pass using python3.  Doing so requires using an updated
starpy and no python sources within tests/ have been updated.  Unit
tests from lib/python can be run against python2 and python3 using
./self_test.

The goal of this is for all tests to continue functioning using python2.
Some tests may work under python3 but individual tests will need to be
addressed separately.  For tests which do not contain a run-test script
the version of python used to execute ./runtests.py will be used:
./runtests.py -t tests/manager/originate
python2 ./runtests.py -t tests/manager/originate
python3 ./runtests.py -t tests/manager/originate

These commands will be python, python2 and python3 respectively.  The
first example where no interpreter is specified uses the shebang from
./runtests.py.  The interpreter used by ./run-local can be controlled
by setting PYTHON environmental variable:
PYTHON=python3 ./run-local run -t tests/manager/originate

Change-Id: If76c2d3e11e4ab4552d0df7841287c8bb2de7918

Author: coreyfarrell

Diff for: lib/python/asterisk/ami.py

@@ -13,7 +13,8 @@
 import logging
 import re
 import json
-from pluggable_registry import PLUGGABLE_EVENT_REGISTRY,\
+from .test_runner import load_and_parse_module
+from .pluggable_registry import PLUGGABLE_EVENT_REGISTRY,\
     PLUGGABLE_ACTION_REGISTRY, var_replace
 
 LOGGER = logging.getLogger(__name__)
@@ -361,7 +362,7 @@ def __init__(self, requirements):
             lower_key = key.lower()
             if lower_key == 'extra':
                 value = dict((key.lower(), value)
-                             for key, value in value.iteritems())
+                             for key, value in value.items())
             self.requirements[lower_key] = value
         self.orderings = requirements.get('partialorder') or []
         self.named_id = requirements.get('id')
@@ -529,8 +530,7 @@ def __init__(self, instance_config, test_object):
 
     def event_callback(self, ami, event):
         """Callback called when an event is received from AMI"""
-        callback_module = __import__(self.callback_module)
-        method = getattr(callback_module, self.callback_method)
+        method = load_and_parse_module(self.callback_module + '.' + self.callback_method)
         self.passed = method(ami, event)
         if self.passed is None:
             LOGGER.error("Callback %s.%s returned None instead of a boolean",
@@ -828,7 +828,7 @@ def __init__(self, test_object, triggered_callback, config):
 
 def replace_ami_vars(mydict, values):
     outdict = {}
-    for key, value in mydict.iteritems():
+    for key, value in mydict.items():
         outdict[key] = var_replace(value, values)
 
     return outdict

Diff for: lib/python/asterisk/apptest.py

@@ -18,8 +18,8 @@
 from twisted.internet import reactor, defer
 
 sys.path.append("lib/python")
-from test_case import TestCase
-from ami import AMIEventInstance
+from .test_case import TestCase
+from .ami import AMIEventInstance
 
 LOGGER = logging.getLogger(__name__)
 
@@ -165,7 +165,7 @@ def _is_scenario_done(self):
         A Scenario is considered done if all results have been met and
         all expected actions have been executed.
         """
-        return (all(self._expected_results.itervalues()) and
+        return (all(self._expected_results.values()) and
                 all(i.ran_actions or i.unexpected
                     for i in self._event_instances))
 

Diff for: lib/python/asterisk/ari.py

@@ -12,12 +12,16 @@
 import re
 import requests
 import traceback
-import urllib
+try:
+    from urllib.parse import urlencode
+except:
+    from urllib import urlencode
 
-from test_case import TestCase
-from pluggable_registry import PLUGGABLE_EVENT_REGISTRY,\
+from .test_case import TestCase
+from .test_runner import load_and_parse_module
+from .pluggable_registry import PLUGGABLE_EVENT_REGISTRY,\
     PLUGGABLE_ACTION_REGISTRY, var_replace
-from test_suite_utils import all_match
+from .test_suite_utils import all_match
 from twisted.internet import reactor
 try:
     from autobahn.websocket import WebSocketClientFactory, \
@@ -332,7 +336,7 @@ def __init__(self, receiver, host, apps, userpass, port=DEFAULT_PORT,
         """
         url = "ws://%s:%d/ari/events?%s" % \
               (host, port,
-               urllib.urlencode({'app': apps, 'api_key': '%s:%s' % userpass}))
+               urlencode({'app': apps, 'api_key': '%s:%s' % userpass}))
         if subscribe_all:
             url += '&subscribeAll=true'
         LOGGER.info("WebSocketClientFactory(url=%s)", url)
@@ -560,7 +564,7 @@ def send(self, values):
         url = self.ari.build_url(uri)
         requests_method = getattr(requests, self.method)
         params = dict((key, var_replace(val, values))
-                      for key, val in self.params.iteritems())
+                      for key, val in self.params.items())
 
         response = requests_method(
             url,
@@ -580,7 +584,7 @@ def send(self, values):
                              response.status_code, response.text)
                 return False
         else:
-            if response.status_code / 100 != 2:
+            if response.status_code // 100 != 2:
                 LOGGER.error('sent %s %s %s response %d %s',
                              self.method, self.uri, self.params,
                              response.status_code, response.text)
@@ -606,8 +610,7 @@ def __init__(self, ari, instance_config, test_object):
 
         callback = self.instance_config.get('callback')
         if callback:
-            module = __import__(callback['module'])
-            self.callback = getattr(module, callback['method'])
+            self.callback = load_and_parse_module(callback['module'] + '.' + callback['method'])
         else:
             # No callback; just use a no-op
             self.callback = lambda *args, **kwargs: True

Diff for: lib/python/asterisk/astconfigparser.py

@@ -8,8 +8,7 @@
 import re
 import itertools
 
-from astdicts import OrderedDict
-from astdicts import MultiOrderedDict
+from .astdicts import OrderedDict, MultiOrderedDict
 
 
 def merge_values(left, right, key):
@@ -270,11 +269,11 @@ def found(d):
 
 def write_dicts(config_file, mdicts):
     """Write the contents of the mdicts to the specified config file"""
-    for section, sect_list in mdicts.iteritems():
+    for section, sect_list in mdicts.items():
         # every section contains a list of dictionaries
         for sect in sect_list:
             config_file.write("[%s]\n" % section)
-            for key, val_list in sect.iteritems():
+            for key, val_list in sect.items():
                 # every value is also a list
                 for v in val_list:
                     key_val = key
@@ -355,7 +354,7 @@ def get_sections(self, key, attr='_sections', searched=None):
         if self._includes:
             res.extend(list(itertools.chain(*[
                 incl.get_sections(key, attr, searched)
-                for incl in self._includes.itervalues()])))
+                for incl in self._includes.values()])))
         if self._parent:
             res += self._parent.get_sections(key, attr, searched)
         return res
@@ -445,7 +444,7 @@ def read(self, filename, sect=None):
             with open(filename, 'rt') as config_file:
                 self._read(config_file, sect)
         except IOError:
-            print "Could not open file ", filename, " for reading"
+            print("Could not open file ", filename, " for reading")
 
     def _read(self, config_file, sect):
         """Parse configuration information from the config_file"""
@@ -478,7 +477,7 @@ def _read(self, config_file, sect):
     def write(self, config_file):
         """Write configuration information out to a file"""
         try:
-            for key, val in self._includes.iteritems():
+            for key, val in self._includes.items():
                 val.write(key)
                 config_file.write('#include "%s"\n' % key)
 
@@ -490,4 +489,4 @@ def write(self, config_file):
                 with open(config_file, 'wt') as fp:
                     self.write(fp)
             except IOError:
-                print "Could not open file ", config_file, " for writing"
+                print("Could not open file ", config_file, " for writing")

Diff for: lib/python/asterisk/astcsv.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """Asterisk CSV-based testing
 
 This module implements the basic CSV testing backend for things like
@@ -11,7 +10,6 @@
 the GNU General Public License Version 2.
 """
 
-import unittest
 import sys
 import csv
 import re
@@ -57,10 +55,10 @@ def match(self, other, silent=False, exact=(False, False)):
         else:
             cmp_fn = (lambda x, y: str(x).lower() == str(y).lower())
 
-        for key, value in self.iteritems():
+        for key, value in self.items():
             if None not in (value, other.get(key)) and not cmp_fn(value, other.get(key)):
                 if not silent:
-                    LOGGER.warn("CSV MATCH FAILED, Expected: %s: '%s' "
+                    LOGGER.warning("CSV MATCH FAILED, Expected: %s: '%s' "
                                 "Got: %s: '%s'" % (key, value, key,
                                                    other.get(key)))
                 return False
@@ -70,9 +68,9 @@ def get(self, key):
         """Retrieve a value from the specified column key"""
         return self.__columns.get(key)
 
-    def iteritems(self):
+    def items(self):
         """Iterate over the values in the columns"""
-        return self.__columns.iteritems()
+        return self.__columns.items()
 
     def __str__(self):
         return ",".join(["\"%s\"" % (self.__dict__[x]) for x in self.__fields])
@@ -94,12 +92,14 @@ def __init__(self, fname=None, records=None, fields=None, row_factory=None):
         self.__records = []
 
         csvreader = None
+        csvfile = None
 
         try:
-            csvreader = csv.DictReader(open(self.filename, "r"), fields, ",")
-        except IOError as (errno, strerror):
+            csvfile = open(self.filename, "r")
+            csvreader = csv.DictReader(csvfile, fields, ",")
+        except IOError as e:
             LOGGER.error("IOError %d[%s] while opening file '%s'" %
-                         (errno, strerror, self.filename))
+                         (e.errno, e.strerror, self.filename))
         except:
             LOGGER.error("Unexpected error: %s" % (sys.exc_info()[0]))
 
@@ -111,6 +111,8 @@ def __init__(self, fname=None, records=None, fields=None, row_factory=None):
             record = self.row_factory(**row)
             self.__records.append(record)
 
+        csvfile.close()
+
     def __len__(self):
         return len(self.__records)
 
@@ -125,7 +127,7 @@ def match(self, other, partial=False):
         each record"""
 
         if not partial and (len(self) != len(other)):
-            LOGGER.warn("CSV MATCH FAILED, different number of records, "
+            LOGGER.warning("CSV MATCH FAILED, different number of records, "
                         "self=%d and other=%d" % (len(self), len(other)))
             return False
 
@@ -147,7 +149,7 @@ def match_order(list_a, list_b, cmp_func):
             size = len(list_a)
 
             # attempt two orderings: forward and reversed
-            guess_orders = (range(size), list(reversed(range(size))))
+            guess_orders = (list(range(size)), list(reversed(range(size))))
             found_orders = []
 
             for guess_order in guess_orders:
@@ -185,15 +187,15 @@ def match_order(list_a, list_b, cmp_func):
             # have it complain immediately.
             for i, item in enumerate(self):
                 if not item.match(other[i], exact=exactness):
-                    LOGGER.warn("Failed to match entry %d" % (i,))
+                    LOGGER.warning("Failed to match entry %d" % (i,))
                     return False
             assert False
 
         elif len(matches) == 1:
             pass  # joy!
 
         elif len(matches) > 1:
-            LOGGER.warn("More than one CSV permutation results in success")
+            LOGGER.warning("More than one CSV permutation results in success")
 
         return True
 
@@ -205,9 +207,6 @@ def empty(self):
         try:
             open(self.filename, "w").close()
         except:
-            LOGGER.warn("Unable to empty CSV file %s" % (self.filename))
-
-if __name__ == '__main__':
-    unittest.main()
+            LOGGER.warning("Unable to empty CSV file %s" % (self.filename))
 
 # vim:sw=4:ts=4:expandtab:textwidth=79