prepare for py3 by replacing obsoleted string.letters with backwardscompatible string.ascii_letters. Fix codegrep etc. to work on files relative to 'mig' code root dir again

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@4898 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

addheader.py

@@ -35,7 +35,8 @@
 import os
 import sys
 
-from mig.shared.projcode import py_code_files, sh_code_files, js_code_files
+from mig.shared.projcode import code_root, py_code_files, sh_code_files, \
+    js_code_files
 
 # Modify these to fit actual project
 proj_vars = {}
@@ -185,7 +186,7 @@ def add_header(path, var_dict, explicit_border=True, block_wrap=False):
                 else:
                     needs_block = False
 
-                pattern = os.path.join(mig_code_base, pattern)
+                pattern = os.path.join(mig_code_base, code_root, pattern)
 
                 # print "Testing %s against %s" % (src_path, pattern)
 

bugweed.py

@@ -31,9 +31,10 @@
 from __future__ import absolute_import
 
 import glob
+import os
 import sys
 
-from mig.shared.projcode import py_code_files
+from mig.shared.projcode import code_root, py_code_files
 from mig.shared.safeeval import subprocess_call
 
 if '__main__' == __name__:
@@ -42,9 +43,11 @@
         print('Grep for obvious errors in all code files')
         sys.exit(1)
 
+    mig_code_base = os.path.dirname(sys.argv[0])
     expanded_paths = []
     for code_path in py_code_files:
-        expanded_paths += glob.glob(code_path)
+        path_pattern = os.path.join(mig_code_base, code_root, code_path)
+        expanded_paths += glob.glob(os.path.normpath(path_pattern))
     command_list = ["pylint", "-E"] + expanded_paths
     command = ' '.join(command_list)
     print("Bug weeding command: %s" % command)

codegrep.py

@@ -32,9 +32,10 @@
 from __future__ import absolute_import
 
 import glob
+import os
 import sys
 
-from mig.shared.projcode import code_files
+from mig.shared.projcode import code_root, code_files
 from mig.shared.safeeval import subprocess_call
 
 
@@ -44,10 +45,12 @@
         print('Grep for PATTERN in all code files')
         sys.exit(1)
 
+    mig_code_base = os.path.dirname(sys.argv[0])
     pattern = sys.argv[1]
     expanded_paths = []
     for code_path in code_files:
-        expanded_paths += glob.glob(code_path)
+        path_pattern = os.path.join(mig_code_base, code_root, code_path)
+        expanded_paths += glob.glob(os.path.normpath(path_pattern))
     command_list = ["grep", "-E", "%s" % pattern] + expanded_paths
     # NOTE: we use command list to avoid shell requirement
     subprocess_call(command_list)

mig/shared/findtype.py

@@ -4,7 +4,7 @@
 # --- BEGIN_HEADER ---
 #
 # findtype - Detect client entity type
-# Copyright (C) 2003-2018  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2020  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -29,15 +29,15 @@
 from __future__ import absolute_import
 
 import os
-from string import letters, digits
+from string import ascii_letters, digits
 
 from mig.shared.defaults import user_db_filename
 from mig.shared.base import client_id_dir
 from mig.shared.listhandling import is_item_in_pickled_list
 from mig.shared.validstring import valid_user_path
 from mig.shared.serial import load
 
-VALID_FQDN_CHARACTERS = letters + digits + '.-'
+VALID_FQDN_CHARACTERS = ascii_letters + digits + '.-'
 MIG_SERVER_ID = 'MiG-Server'
 
 

mig/shared/mrsltoxrsl.py

@@ -1,12 +1,11 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-
 #
 # --- BEGIN_HEADER ---
 #
 #
 # mrsltoxrsl - [optionally add short module description on this line]
-# Copyright (C) 2003-2017  The MiG Project lead by Brian Vinter
+# Copyright (C) 2003-2020  The MiG Project lead by Brian Vinter
 #
 # This file is part of MiG.
 #
@@ -97,10 +96,10 @@ def indent(acc, n, s):
         if not s:
             return acc
         if s[0] == '(':
-            start = '\n' + ' '*n
-            return(indent(acc + start + '(', n+2, s[1:]))
+            start = '\n' + ' ' * n
+            return(indent(acc + start + '(', n + 2, s[1:]))
         elif s[0] == ')':
-            return(indent(acc + ')', n-2, s[1:]))
+            return(indent(acc + ')', n - 2, s[1:]))
 
         return(indent(acc + s[0], n, s[1:]))
 
@@ -111,7 +110,7 @@ def indent(acc, n, s):
 #              -> (Xrsl,Job Script,name for Job script)
 def translate(mrsl_dict, session_id=None):
     """Translate an (already checked) mRSL dictionary into xRSL,
-       suitable for submitting to an ARC resource. 
+       suitable for submitting to an ARC resource.
 
        Returns arclib.Xrsl object.
        Throws exception if errors in the xRSL generation occur."""
@@ -167,10 +166,10 @@ def translate(mrsl_dict, session_id=None):
             j_name = mrsl_dict['JOB_ID']
         else:
             # random string. should not happen anyway...
-            j_name = ''.join(random.choice(string.letters)
+            j_name = ''.join(random.choice(string.ascii_letters)
                              for i in xrange(12))
 #        j_name = mrsl_dict.get('JOB_ID',
-#                               ''.join(random.choice(string.letters)  \
+#                               ''.join(random.choice(string.ascii_letters) \
 #                                       for i in xrange(12)))
 
         # use JOBID as ARC jobname to avoid presenting only ARC IDs
@@ -247,7 +246,7 @@ def fst(list):
         # requested runtime, given in minutes in (user) xrsl ...
         time = mrsl_dict.get('CPUTIME')
         if time:
-            addRel(xrsl, 'cputime', str(int(math.ceil(float(time)/60))))
+            addRel(xrsl, 'cputime', str(int(math.ceil(float(time) / 60))))
 
         # simply copy the values for these:
         copy_items = ['MEMORY', 'DISK', 'NODECOUNT']
@@ -286,8 +285,8 @@ def fst(list):
                 if address != 'SETTINGS':
                     addresses.append(address)
 #                else:
-#                    # this should be replaced already, but...
-#                    # FIXME: get it from the settings :-P
+# this should be replaced already, but...
+# FIXME: get it from the settings :-P
 #                    addresses.append('*FROM THE SETTINGS*')
             if addresses:
                 addRel(xrsl, 'notify', 'ec ' + ' '.join(addresses))
@@ -354,7 +353,7 @@ def is_mail(str):
 
 
 def file_mapping(line):
-    """Splits the given line of the expected format 
+    """Splits the given line of the expected format
           local_name <space> remote_name
        into a 2-element list [local_name,remote_name]
        If remote_name is empty, the empty string is returned as the 2nd part.

mig/shared/projcode.py

@@ -23,9 +23,14 @@
 
 """Helpers to transform or search project code files"""
 
+# Top dir with all code
+code_root = 'mig'
+
 # Ignore backup and dot files in wild card match
 plain = '[a-zA-Z0-9]*.py'
 py_code_files = [
+    # a few scripts are in parent dir of code_root
+    '../%s' % plain,
     '%s' % plain,
     'cgi-bin/%s' % plain,
     'cgi-sid/%s' % plain,

mig/shared/safeinput.py

@@ -39,7 +39,7 @@
 
 import cgi
 from email.utils import parseaddr, formataddr
-from string import letters, digits, printable
+from string import ascii_letters, digits, printable
 from unicodedata import category, normalize, name as unicode_name
 
 from mig.shared.base import force_unicode, force_utf8
@@ -105,16 +105,16 @@
 
 # We must be careful about characters that have special regex meaning
 
-VALID_SAFE_PATH_CHARACTERS = letters + digits + "/.,_-+="
-VALID_PATH_CHARACTERS = letters + digits + CURRENCY + "/.,_-+±×÷=½¾" + \
+VALID_SAFE_PATH_CHARACTERS = ascii_letters + digits + "/.,_-+="
+VALID_PATH_CHARACTERS = ascii_letters + digits + CURRENCY + "/.,_-+±×÷=½¾" + \
     " " + "'" + ":;@§%‰()~!&¶"
 
 # Plain text here only - *no* html tags, i.e. no '<' or '>' !!
 
 VALID_TEXT_CHARACTERS = VALID_PATH_CHARACTERS + CURRENCY + '?#*[]{}' + '"' + \
     "`|^" + '\\' + '\n\r\t'
-VALID_FQDN_CHARACTERS = letters + digits + '.-'
-VALID_BACKEND_NAME_CHARACTERS = letters + digits + '-_'
+VALID_FQDN_CHARACTERS = ascii_letters + digits + '.-'
+VALID_BACKEND_NAME_CHARACTERS = ascii_letters + digits + '-_'
 VALID_BASEURL_CHARACTERS = VALID_FQDN_CHARACTERS + ':/_'
 VALID_URL_CHARACTERS = VALID_BASEURL_CHARACTERS + '?;&%='
 # According to https://tools.ietf.org/html/rfc3986#section-2 URLs may contain
@@ -138,15 +138,15 @@
 # Allow these chars in addition to plain letters and digits
 # We explicitly allow email chars in CN to work around broken DNs
 
-#############################################################################
-# IMPORTANT: never allow '+' or '_' in name: reserved for path translation! #
-#############################################################################
+#*****************************************************************************
+#* IMPORTANT: never allow '+' or '_' in name: reserved for path translation! *
+#*****************************************************************************
 
 name_extras = ' -@.'
 
-#############################################################################
-# IMPORTANT: never allow '+' in DN: reserved for path translation!          #
-#############################################################################
+#*****************************************************************************
+#* IMPORTANT: never allow '+' in DN: reserved for path translation!          *
+#*****************************************************************************
 # We allow ':' in DN, however, as it is used by e.g. DanID:
 # /C=DK/O=Ingen organisatorisk tilknytning/CN=${NAME}/serialNumber=PID:${SERIAL}
 # Similarly we must allow '_' in DN since it is valid in emailAddress. We only
@@ -165,9 +165,9 @@
 
 valid_integer_chars = digits + integer_extras
 valid_float_chars = digits + float_extras
-valid_password_chars = letters + digits + password_extras
-valid_name_chars = letters + digits + name_extras
-valid_dn_chars = letters + digits + dn_extras
+valid_password_chars = ascii_letters + digits + password_extras
+valid_name_chars = ascii_letters + digits + name_extras
+valid_dn_chars = ascii_letters + digits + dn_extras
 valid_username_chars = user_id_charset
 VALID_INTEGER_CHARACTERS = valid_integer_chars
 VALID_FLOAT_CHARACTERS = valid_float_chars
@@ -305,7 +305,8 @@ def valid_printable(contents, min_length=0, max_length=-1):
 def valid_ascii(contents, min_length=0, max_length=-1, extra_chars=''):
     """Verify that supplied contents only contain ascii characters"""
 
-    __valid_contents(contents, letters + extra_chars, min_length, max_length)
+    __valid_contents(
+        contents, ascii_letters + extra_chars, min_length, max_length)
 
 
 def valid_numeric(contents, min_length=0, max_length=-1):
@@ -317,7 +318,8 @@ def valid_numeric(contents, min_length=0, max_length=-1):
 def valid_alphanumeric(contents, min_length=0, max_length=-1, extra_chars=''):
     """Verify that supplied contents only contain alphanumeric characters"""
 
-    __valid_contents(contents, letters + digits + extra_chars, min_length,
+    __valid_contents(
+        contents, ascii_letters + digits + extra_chars, min_length,
                      max_length)
 
 
@@ -814,7 +816,7 @@ def __valid_gdp_var(gdp_var):
     """Verify that supplied gdp_var only contains characters that
     we consider valid in various gdp variable names.
     """
-    __valid_contents(gdp_var, letters + '_')
+    __valid_contents(gdp_var, ascii_letters + '_')
 
 
 def valid_gdp_category_id(category_id):
@@ -835,7 +837,7 @@ def valid_gdp_ref_value(ref_value):
     """Verify that supplied ref_value only contains characters that
     we consider valid in GDP ref values.
     """
-    __valid_contents(ref_value, letters + digits + '+-=/.:_')
+    __valid_contents(ref_value, ascii_letters + digits + '+-=/.:_')
 
 
 def valid_cloud_instance_id(
@@ -1026,8 +1028,9 @@ def valid_workflow_attributes(attributes):
 
 
 def valid_workflow_operation(operation):
-    """Verify that the supplied workflow
-    operation only contains letters + _ """
+    """Verify that the supplied workflow operation only contains letters and
+    underscores.
+    """
     valid_ascii(operation, extra_chars='_')
 
 
@@ -1055,19 +1058,23 @@ def valid_job_attributes(attributes):
 
 
 def valid_job_type(type):
-    """Verify that the supplied job type only contains letters + _ """
+    """Verify that the supplied job type only contains letters and
+    underscores.
+    """
     valid_ascii(type, extra_chars='_')
 
 
 def valid_job_operation(operation):
-    """Verify that the supplied job operation only contains letters + _ """
+    """Verify that the supplied job operation only contains letters and
+    underscores.
+    """
     valid_ascii(operation, extra_chars='_')
 
 
 def filter_ascii(contents):
     """Filter supplied contents to only contain ascii characters"""
 
-    return __filter_contents(contents, letters)
+    return __filter_contents(contents, ascii_letters)
 
 
 def filter_numeric(contents):
@@ -1079,13 +1086,13 @@ def filter_numeric(contents):
 def filter_alphanumeric(contents):
     """Filter supplied contents to only contain alphanumeric characters"""
 
-    return __filter_contents(contents, letters + digits)
+    return __filter_contents(contents, ascii_letters + digits)
 
 
 def filter_alphanumeric_and_spaces(contents):
     """Filter supplied contents to only contain alphanumeric characters"""
 
-    return __filter_contents(contents, letters + digits + ' ')
+    return __filter_contents(contents, ascii_letters + digits + ' ')
 
 
 def filter_date(contents):