Merge pull request #160 from autolab/python3-upgrade

Python 3 Upgrade

Author: fanpu

Diff for: .gitignore

@@ -22,3 +22,8 @@ pip-selfcheck.json
 
 # IDEs
 .idea
+.vscode
+
+# Backup files
+*.bak
+

Diff for: README.md

@@ -35,8 +35,10 @@ Please feel free to use Tango at your school/organization. If you run into any p
 3. [Read the documentation for the VMMS API](https://github.com/autolab/Tango/wiki/Tango-VMMS-API).
 4. [Test whether Tango is set up properly and can process jobs](https://github.com/autolab/Tango/wiki/Testing-Tango).
 
-## Python 3 Upgrade
-We are in the process of porting Tango from Python 2 to Python 3. The current working branch for the update is `python3-upgrade`.
+## Python 2 Support
+Tango now runs on Python 3. However, there is a legacy branch [master-python2](https://github.com/autolab/Tango/tree/master-python2) which is a snapshot of the last Python 2 Tango commit for legacy reasons. You are strongly encouraged to upgrade to the current Python 3 version of Tango if you are still on the Python 2 version, as future enhancements and bug fixes will be focused on the current master. 
+
+We will not be backporting new features from `master` to `master-python2`.
 
 ## Contributing to Tango
 

Diff for: clients/tango-cli.py

@@ -4,6 +4,12 @@
 # tango-cli.py - Command line client for the RESTful Tango.
 #
 
+from __future__ import print_function
+from builtins import range
+from future import standard_library
+standard_library.install_aliases()
+from builtins import map
+from builtins import str
 import os
 import sys
 
@@ -12,7 +18,7 @@
 import argparse
 import requests
 import json
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 #
 #
@@ -95,35 +101,35 @@
 
 def checkKey():
     if (args.key is None):
-        print "Key must be specified with -k"
+        print("Key must be specified with -k")
         return -1
     return 0
 
 
 def checkCourselab():
     if (args.courselab is None):
-        print "Courselab must be specified with -l"
+        print("Courselab must be specified with -l")
         return -1
     return 0
 
 
 def checkFilename():
     if (args.filename is None):
-        print "Filename must be specified with --filename"
+        print("Filename must be specified with --filename")
         return -1
     return 0
 
 
 def checkInfiles():
     if (args.infiles is None):
-        print "Input files must be specified with --infiles"
+        print("Input files must be specified with --infiles")
         return -1
     return 0
 
 
 def checkDeadjobs():
     if (args.deadJobs is None):
-        print "Deadjobs must be specified with --deadJobs"
+        print("Deadjobs must be specified with --deadJobs")
         return -1
     return 0
 
@@ -139,11 +145,11 @@ def tango_open():
         response = requests.get(
             'http://%s:%d/open/%s/%s/' %
             (args.server, args.port, args.key, args.courselab))
-        print "Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)
-        print response.content
+        print("Sent request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab)
+        print("Failed to send request to %s:%d/open/%s/%s/" % (args.server, args.port, args.key, args.courselab))
         print (str(err))
         sys.exit(0)
 
@@ -170,11 +176,11 @@ def tango_upload():
             data=f.read(),
             headers=header)
         f.close()
-        print "Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)
-        print response.content
+        print("Sent request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename)
+        print("Failed to send request to %s:%d/upload/%s/%s/ filename=%s" % (args.server, args.port, args.key, args.courselab, args.filename))
         print (str(err))
         sys.exit(0)
 
@@ -208,11 +214,11 @@ def tango_addJob():
              args.key,
              args.courselab),
             data=json.dumps(requestObj))
-        print "Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))
-        print response.content
+        print("Sent request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj))
+        print("Failed to send request to %s:%d/addJob/%s/%s/ \t jobObj=%s" % (args.server, args.port, args.key, args.courselab, json.dumps(requestObj)))
         print (str(err))
         sys.exit(0)
 
@@ -231,13 +237,13 @@ def tango_poll():
              args.port,
              args.key,
              args.courselab,
-             urllib.quote(
+             urllib.parse.quote(
                  args.outputFile)))
-        print "Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))
-        print response.content
+        print("Sent request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile)))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.quote(args.outputFile))
+        print("Failed to send request to %s:%d/poll/%s/%s/%s/" % (args.server, args.port, args.key, args.courselab, urllib.parse.quote(args.outputFile)))
         print (str(err))
         sys.exit(0)
 
@@ -252,11 +258,11 @@ def tango_info():
 
         response = requests.get(
             'http://%s:%d/info/%s/' % (args.server, args.port, args.key))
-        print "Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key)
-        print response.content
+        print("Sent request to %s:%d/info/%s/" % (args.server, args.port, args.key))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key)
+        print("Failed to send request to %s:%d/info/%s/" % (args.server, args.port, args.key))
         print (str(err))
         sys.exit(0)
 
@@ -272,11 +278,11 @@ def tango_jobs():
         response = requests.get(
             'http://%s:%d/jobs/%s/%d/' %
             (args.server, args.port, args.key, args.deadJobs))
-        print "Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)
-        print response.content
+        print("Sent request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs)
+        print("Failed to send request to %s:%d/jobs/%s/%d/" % (args.server, args.port, args.key, args.deadJobs))
         print (str(err))
         sys.exit(0)
 
@@ -291,11 +297,11 @@ def tango_pool():
 
         response = requests.get('http://%s:%d/pool/%s/%s/' %
                                 (args.server, args.port, args.key, args.image))
-        print "Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)
-        print response.content
+        print("Sent request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image)
+        print("Failed to send request to %s:%d/pool/%s/%s/" % (args.server, args.port, args.key, args.image))
         print (str(err))
         sys.exit(0)
 
@@ -321,11 +327,11 @@ def tango_prealloc():
              args.image,
              args.num),
             data=json.dumps(vmObj))
-        print "Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))
-        print response.content
+        print("Sent request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)))
+        print(response.text)
 
     except Exception as err:
-        print "Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj))
+        print("Failed to send request to %s:%d/prealloc/%s/%s/%s/ \t vmObj=%s" % (args.server, args.port, args.key, args.image, args.num, json.dumps(vmObj)))
         print (str(err))
         sys.exit(0)
 
@@ -343,31 +349,31 @@ def file_to_dict(file):
 
 def tango_runJob():
     if args.runJob is None:
-        print "Invalid usage: [runJob]"
+        print("Invalid usage: [runJob]")
         sys.exit(0)
 
     dir = args.runJob
     infiles = [file for file in os.listdir(
         dir) if os.path.isfile(os.path.join(dir, file))]
     files = [os.path.join(dir, file) for file in infiles]
-    args.infiles = map(file_to_dict, infiles)
+    args.infiles = list(map(file_to_dict, infiles))
 
     args.jobname += "-0"
     args.outputFile += "-0"
-    for i in xrange(1, args.numJobs + 1):
-        print "----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------"
-        print "----------- OPEN"
+    for i in range(1, args.numJobs + 1):
+        print("----------------------------------------- STARTING JOB " + str(i) + " -----------------------------------------")
+        print("----------- OPEN")
         tango_open()
-        print "----------- UPLOAD"
+        print("----------- UPLOAD")
         for file in files:
             args.filename = file
             tango_upload()
-        print "----------- ADDJOB"
+        print("----------- ADDJOB")
         length = len(str(i - 1))
         args.jobname = args.jobname[:-length] + str(i)
         args.outputFile = args.outputFile[:-length] + str(i)
         tango_addJob()
-        print "--------------------------------------------------------------------------------------------------\n"
+        print("--------------------------------------------------------------------------------------------------\n")
 
 
 def router():
@@ -403,7 +409,7 @@ def router():
 try:
     response = requests.get('http://%s:%d/' % (args.server, args.port))
 except:
-    print 'Tango not reachable on %s:%d!\n' % (args.server, args.port)
+    print('Tango not reachable on %s:%d!\n' % (args.server, args.port))
     sys.exit(0)
 
 router()

Diff for: config.template.py

@@ -2,12 +2,13 @@
 # config.py - Global configuration constants and runtime info
 #
 
+from builtins import object
 import logging, time
 
 # Config - defines
 
 
-class Config:
+class Config(object):
     #####
     # Part 1: Tango constants for developers
     #

Diff for: jobManager.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 #
 # JobManager - Thread that assigns jobs to worker threads
 #
@@ -9,6 +10,10 @@
 # is launched that will handle things from here on. If anything goes
 # wrong, the job is made dead with the error.
 #
+from builtins import object
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
 import threading, logging, time, copy
 
 from datetime import datetime
@@ -20,7 +25,7 @@
 from tangoObjects import TangoQueue
 from config import Config
 
-class JobManager:
+class JobManager(object):
 
     def __init__(self, queue):
         self.daemon = True
@@ -62,9 +67,16 @@ def __manage(self):
 
             if id:
                 job = self.jobQueue.get(id)
+
+                # job could no longer exist if it was completed by someone else
+                if job == None:
+                    continue
+
                 if not job.accessKey and Config.REUSE_VMS:
                     id, vm = self.jobQueue.getNextPendingJobReuse(id)
                     job = self.jobQueue.get(id)
+                    if job == None:
+                        continue
 
                 try:
                     # Mark the job assigned

Diff for: jobQueue.py

@@ -7,6 +7,9 @@
 # JobManager: Class that creates a thread object that looks for new
 # work on the job queue and assigns it to workers.
 #
+from builtins import range
+from builtins import object
+from builtins import str
 import threading, logging, time
 
 from datetime import datetime
@@ -28,7 +31,7 @@
 #
 
 
-class JobQueue:
+class JobQueue(object):
 
     def __init__(self, preallocator):
         self.liveJobs = TangoDictionary("liveJobs")
@@ -53,7 +56,7 @@ def _getNextID(self):
         keys = self.liveJobs.keys()
         if (str(id) in keys):
             id = -1
-            for i in xrange(1, Config.MAX_JOBID + 1):
+            for i in range(1, Config.MAX_JOBID + 1):
                 if (str(i) not in keys):
                     id = i
                     break
@@ -191,7 +194,7 @@ def getNextPendingJob(self):
         Called by JobManager when Config.REUSE_VMS==False
         """
         self.queueLock.acquire()
-        for id, job in self.liveJobs.iteritems():
+        for id, job in self.liveJobs.items():
             if job.isNotAssigned():
                 self.queueLock.release()
                 return id
@@ -203,7 +206,7 @@ def getNextPendingJobReuse(self, target_id=None):
         Called by JobManager when Config.REUSE_VMS==True
         """
         self.queueLock.acquire()
-        for id, job in self.liveJobs.iteritems():
+        for id, job in self.liveJobs.items():
             # if target_id is set, only interested in this id
             if target_id and target_id != id:
                 continue

Diff for: preallocator.py

@@ -1,6 +1,8 @@
 #
 # preallocator.py - maintains a pool of active virtual machines
 #
+from builtins import object
+from builtins import range
 import threading, logging, time, copy
 
 from tangoObjects import TangoDictionary, TangoQueue, TangoIntValue
@@ -17,7 +19,7 @@
 #
 
 
-class Preallocator:
+class Preallocator(object):
 
     def __init__(self, vmms):
         self.machines = TangoDictionary("machines")

Diff for: requirements.txt

@@ -1,10 +1,9 @@
-backports.ssl-match-hostname==3.4.0.2
-boto==2.27.0
-futures==2.2.0
-plumbum==1.4.2
-pyflakes==0.8.1
-redis==2.10.3
-requests==2.2.1
-rpyc==3.3.0
-wsgiref==0.1.2
-tornado==4.1
+backports.ssl-match-hostname==3.7.0.1
+boto==2.49.0 # used only by ec2SSH.py
+plumbum==1.6.9
+pyflakes==2.1.1
+redis==3.4.1
+requests==2.23.0
+rpyc==4.1.4
+tornado==4.5.3
+future==0.18.2