Use the correct force_utf8 function based on Python version.

Import the experimental version of force_utf8 wholesale with a py ver
specific suffix and expose the correct implementation dependent on PY2.
Import the experimental breanch version of force_utf8 wholesale adding a
python specific suffix and expose the correct implementation dependent on PY2.

Include making use of forcing InputException messages to a native string
(also taken directly from experimental) which ensures strings are handed
back on __str__() on InputException thus safeinput tests pass everywhere.

Author: albu-diku

mig/shared/base.py

@@ -36,6 +36,7 @@
 import re
 
 # IMPORTANT: do not import any other MiG modules here - to avoid import loops
+from mig.shared.compat import PY2
 from mig.shared.defaults import default_str_coding, default_fs_coding, \
     keyword_all, keyword_auto, sandbox_names, _user_invisible_files, \
     _user_invisible_dirs, _vgrid_xgi_scripts, cert_field_order, csrf_field, \
@@ -496,7 +497,7 @@ def is_unicode(val):
     return (type(u"") == type(val))
 
 
-def force_utf8(val, highlight=''):
+def _force_utf8_py2(val, highlight=''):
     """Internal helper to encode unicode strings to utf8 version. Actual
     changes are marked out with the highlight string if given.
     """
@@ -507,6 +508,31 @@ def force_utf8(val, highlight=''):
         return val
     return "%s%s%s" % (highlight, val.encode("utf8"), highlight)
 
+def _force_utf8_py3(val, highlight='', stringify=True):
+    """Internal helper to encode unicode strings to utf8 version. Actual
+    changes are marked out with the highlight string if given.
+    The optional stringify turns ALL values including numbers into string.
+    """
+    # We run into all kind of nasty encoding problems if we mix
+    if not isinstance(val, basestring):
+        if stringify:
+            val = "%s" % val
+        else:
+            return val
+    if not is_unicode(val):
+        return val
+    if is_unicode(highlight):
+        hl_utf = highlight.encode("utf8")
+    else:
+        hl_utf = highlight
+    return (b"%s%s%s" % (hl_utf, val.encode("utf8"), hl_utf))
+
+
+if PY2:
+    force_utf8 = _force_utf8_py2
+else:
+    force_utf8 = _force_utf8_py3
+
 
 def force_utf8_rec(input_obj, highlight=''):
     """Recursive object conversion from unicode to utf8: useful to convert e.g.

mig/shared/safeinput.py

@@ -58,7 +58,7 @@
     from html import escape as escape_html
 assert escape_html is not None
 
-from mig.shared.base import force_unicode, force_utf8
+from mig.shared.base import force_unicode, force_native_str
 from mig.shared.defaults import src_dst_sep, username_charset, \
     username_max_length, session_id_charset, session_id_length, \
     subject_id_charset, subject_id_min_length, subject_id_max_length, \
@@ -2294,7 +2294,7 @@ def __init__(self, value):
     def __str__(self):
         """Return string representation"""
 
-        return force_utf8(force_unicode(self.value))
+        return force_native_str(self.value)
 
 
 def main(_exit=sys.exit, _print=print):