Merge pull request #133 from hydroshare/develop

 HydroShare-JupyterHub v1.3

.gitignore

@@ -1,6 +1,4 @@
-jupyterhub/env
-jupyterhub/jupyterhub.log
-jupyterhub/jupyterhub.sqlite
-jupyterhub/jupyterhub_cookie_secret
-jupyterhub/test
+**/secrets
+**/*env
+*.swp
 *.pyc

build/celeryworker/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:2.7
+ADD requirements.txt /app/requirements.txt
+
+#ADD ./celeryworker/ /app/utilities
+ADD ./celeryworker/ /app
+#RUN touch /app/utilities/__init__.py
+WORKDIR /app/
+RUN pip install -r requirements.txt
+
+# add docker-cli
+RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-17.09.0-ce.tgz \
+ && tar -xf docker-17.09.0-ce.tgz \
+ && cp docker/docker /usr/bin/docker
+
+ENTRYPOINT celery -A celeryworker worker --loglevel=info

build/celeryworker/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash 
+
+docker build -t cuahsi/celery_executor .

build/celeryworker/celeryworker/__init__.py

@@ -0,0 +1 @@
+from celeryworker import jobs

build/celeryworker/celeryworker/celery.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+from celery import Celery
+import socket
+
+IP = 'rabbit'
+PRJ = 'celeryworker'
+PORT = '5672'
+RABBIT_USER = 'admin'
+RABBIT_PASS = 'mypass'
+app = Celery(PRJ, broker='amqp://%s:%s@%s:%s' % (RABBIT_USER, RABBIT_PASS,
+ IP, PORT), backend='rpc://', include=[
+ 'celeryworker.tasks'])

build/celeryworker/celeryworker/compat.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import, print_function
+import sys, urllib
+is_py2 = sys.version[0] == '2'
+if is_py2:
+    print('Python 2!')
+    import Queue as queue
+    input = raw_input
+    urlencode = urllib.pathname2url
+else:
+    import queue
+    urlencode = urllib.parse.quote

build/celeryworker/celeryworker/jobs.py

@@ -0,0 +1,52 @@
+from __future__ import absolute_import, print_function
+import time
+from .tasks import *
+from .compat import * 
+
+registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook', 'summa']
+
+def get_registered_images():
+
+    task = task_get_registered_images.delay()
+    task = wait_for_task(task)
+    res = task.result.split('\n')
+    imgs = [res[i].split()[0] for i in range(1,len(res)-1) if res[i].split()[0] in registered_images]
+    return imgs
+
+def run(image_name, vol_mount, mount_target, env_vars={}):
+
+    invoker_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()   
+    print('Name: %s' % image_name)
+    print('Local Relative Path: %s' % vol_mount)
+    print('Mount target: %s' % mount_target)
+    print('Invoking Container Id: %s' % invoker_id)
+
+    # make sure the image is registered
+    if image_name not in registered_images:
+        print('Cannot run a non-registered container')
+        return None
+
+    task = task_run_container.delay(image_name, vol_mount, mount_target, invoker_id, env_vars)
+    task = wait_for_task(task)
+    print('task complete')
+    print(task.result)
+    return task
+
+
+
+def sanity_check(string, async=True):
+    if async:
+        task = task.sanity_check(string)
+        res = task.result
+    else:
+        res = task.sanity_check(string)
+    return res
+
+def wait_for_task(task):
+    print('waiting for task to finish', end='')
+    while not task.ready():
+#        print(task.info)
+        print('.', end='')
+        time.sleep(.25)
+    print(' done')
+    return task

build/celeryworker/celeryworker/run_tasks.py

@@ -0,0 +1,42 @@
+from celeryworker.tasks import *
+import time
+import os
+from celeryworker.compat import *
+
+if __name__ == '__main__':
+
+    volume = os.path.abspath('/share')
+    print(volume)
+
+    # get the id of the current container
+    image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
+
+    # all paths should be relative to the mounted volume
+    vars = {'LOCALBASEDIR': 'share/summa_tests',
+            'OUTDIR': 'output/syntheticTestCases/celia1990/',
+            'MASTERPATH': 'settings/syntheticTestCases/celia1990/summa_fileManager_celia1990.txt'
+            }
+
+    res = run_container.delay('summa', 
+                              volume, 
+                              '/tmp/summa', 
+                              image_id, 
+                              vars)
+    while not res.ready():
+        time.sleep(1)
+        print('...working')
+    print(res.result)
+#    print(os.listdir(volume))
+    print('finished')
+
+
+
+#if __name__ == '__main__':
+#    for _ in xrange(10):
+#        result = longtime_add.delay(1,2)
+#        print 'Task finished?',result.ready()
+#        print 'Task result:',result.result
+#        time.sleep(1)
+#        print 'Task finished"',result.ready()
+#        print 'Task result:',result.result
+#

build/celeryworker/celeryworker/tasks.py

@@ -0,0 +1,75 @@
+from __future__ import absolute_import, print_function
+from celeryworker.celery import app
+import time, subprocess, shlex, os
+from celeryworker.compat import *
+
+
+registered_images = ['castrona/jhswarm', 'jupyter/scipy-notebook']
+
+@app.task
+def task_sanity_check(string):
+    time.sleep(1)
+    print(string)
+    image_id = os.popen('basename "$(head /proc/1/cgroup)"').read().strip()
+    print(image_id)
+    time.sleep(1)
+    return {'success': 1,'imageid': image_id}
+
+@app.task
+def task_get_registered_images():
+    res = os.popen("docker images").read()
+    return res
+#    cmd = 'docker images'
+#    res = run_command(cmd)
+#    return res
+#    res = os.popen("docker images").read().split()
+#    imgs = [res[i] for i in range(6,len(res),7) if res[i] in registered_images]
+#    return imgs
+
+@app.task
+def task_run_container(image_name, vol_mount, mount_target, invoker_id, env_vars={}):
+
+#    print('Name: %s' % image_name)
+#    print('Local Relative Path: %s' % vol_mount)
+#    print('Mount target: %s' % mount_target)
+#    print('Invoking Container Id: %s' % invoker_id)
+
+    vols = os.popen("docker inspect -f '{{ .Mounts }}' %s" % invoker_id).read()
+    volumes = []
+    for vol in vols.strip()[2:-2].split('} {'):
+        atts = vol.split(' ')
+        mnt_path = os.path.join(mount_target, atts[3].strip('/'))
+        volumes.append('-v ' + (':').join([atts[2], mnt_path]))
+
+    for v in volumes:
+        print(v)
+
+    envars = '-e %s=%s' % ('RELPATH',
+     os.path.join('/tmp', vol_mount.strip('/')))
+    for k, v in env_vars.items():
+        envars += ' -e %s=%s' % (k, v)
+
+    print(envars)
+    run_cmd = 'docker run --rm %s %s %s' % ((' ').join(volumes),
+     envars,
+     image_name)
+    print('RUN COMMAND: %s' % run_cmd)
+
+    res = os.popen(run_cmd).read()
+    return res
+#    res = run_command(run_cmd)
+#    print('finished.  returning output')
+#    return res
+
+
+def run_command(command):
+    process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
+    while True:
+        output = process.stdout.readline()
+        if output == '' and process.poll() is not None:
+            break
+        if output:
+            print(output.strip())
+
+    rc = process.poll()
+    return rc

build/celeryworker/requirements.txt

@@ -0,0 +1 @@
+celery==4.0.2

build/jupyterhub-rest/Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:xenial
+
+MAINTAINER Tony Castronova <[email protected]>
+
+
+ADD install-deps.sh /tmp/install-deps.sh
+#ADD utilities.py /tmp/utilities.py
+RUN sh /tmp/install-deps.sh
+
+RUN mkdir /app
+WORKDIR /app 
+ADD start.py /app/start.py
+
+ENTRYPOINT python3 start.py
+
+

build/jupyterhub-rest/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t cuahsi/jupyterhub-rest .

build/jupyterhub-rest/install-deps.sh

@@ -0,0 +1,19 @@
+#!/bin/bash 
+set -x
+set -e
+
+apt-get update
+apt-get install --fix-missing -y --no-install-recommends \
+ git \
+ vim \
+ ca-certificates \
+ python3 \
+ python3-pip
+
+pip3 install setuptools wheel
+
+cd /srv
+git clone https://github.com/hydroshare/hydroshare-jupyterhub.git
+#rm hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
+#cp utilities.py hydroshare-jupyterhub/jupyterhub_rest_server/jupyterhub_rest_server/utilities.py
+(cd hydroshare-jupyterhub; git pull; pip3 install -e jupyterhub_rest_server)

build/jupyterhub-rest/start.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import os
+from jupyterhub_rest_server import server
+
+# check that all the required envvars exist
+args = ['HYDRPSHARE_REDIRECT_COOKIE', 'JUPYTER_HUB_IP',
+        'JUPYTER_REST_IP', 'JUPYTER_PORT', 'JUPYTER_REST_PORT',
+        'JUPYTER_USERSPACE_DIR', 'JUPYTER_USER', 'JUPYTER_NOTEBOOK_DIR']
+for a in args:
+    try:
+        v = os.environ[a]
+    except:
+        print('Missing required EnvVar: %s' % a)
+
+# check that the notebook path exists
+dirs = ['JUPYTER_NOTEBOOK_DIR', 'JUPYTER_USERSPACE_DIR']
+for d in dirs:
+    p = os.environ[d]
+    if not os.path.exists(p):
+        print('Path does not exist: %s:%s' % (d, p))
+
+# start the server
+server.main()

build/jupyterhub-rest/utilities.py

@@ -0,0 +1,82 @@
+# this file contains utility functions used by the RequestHandlers
+
+import os, stat
+from pwd import getpwnam
+import grp
+import shutil
+import logging
+
+log = logging.getLogger()
+
+
+def set_hydroshare_args(username, resourceid, resourcetype):
+
+    userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
+    hs_env = os.path.abspath(os.path.join(userspace_dir, '%s/notebooks/.env' % username.lower()))
+    print('ENV_PATH ',hs_env)
+
+    with open(hs_env, 'w') as f:
+        f.write('HS_USR_NAME=%s\n' % username)
+        f.write('HS_RES_ID=%s\n' % resourceid)
+        f.write('HS_RES_TYPE=%s\n' % resourcetype)
+        f.write('JUPYTER_HUB_IP=%s\n' % os.environ['JUPYTER_HUB_IP'] )
+
+    # get the jupyter username
+    user = getpwnam(os.environ['JUPYTER_USER'])
+    group = grp.getgrnam('users')
+    uid = user.pw_uid
+    gid = group.gr_gid
+    os.chown(hs_env, uid, gid)
+
+def build_userspace(username):
+
+    # make all usernames lowercase
+    husername = username.lower()
+
+    # get the jupyter username
+    user = getpwnam(os.environ['JUPYTER_USER'])
+    group = grp.getgrnam('users')
+    uid = user.pw_uid
+    gid = group.gr_gid
+
+    userspace_dir = os.environ['JUPYTER_USERSPACE_DIR']
+    ipynb_dir = os.environ['JUPYTER_NOTEBOOK_DIR']
+
+    # check to see if user exists
+    basepath = os.path.abspath(os.path.join(userspace_dir, '%s'%husername))
+    path = os.path.abspath(os.path.join(basepath, 'notebooks'))
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    file_paths = []
+    print('%s -> copying userpace filse' % username, flush=True)
+    #ipynb_dir = '../jupyter-rest-endpoint/notebooks'
+    for root, dirs, files in os.walk(ipynb_dir):
+        for file in files:
+            file_paths.append(os.path.join(os.path.abspath(root), file))
+    relpaths = [os.path.relpath(p, ipynb_dir) for p in file_paths]
+    for i in range(0, len(file_paths)):
+        src = file_paths[i]
+        dst = os.path.join(path, relpaths[i])
+        dirpath = os.path.dirname(dst)
+        if not os.path.exists(dirpath):
+            os.makedirs(dirpath)
+        shutil.copyfile(src, dst)
+
+    # change file ownership so that it can be accessed inside docker container
+    print('%s -> modifying userspace permissions' % username, flush=True)
+    os.chown(basepath, uid, gid)
+    os.chown(os.path.dirname(basepath), uid, gid)
+#    os.chmod(os.path.dirname(basepath), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+    os.chmod(os.path.dirname(basepath), 0o2770)
+
+    for root, dirs, files in os.walk(basepath):
+        for d in dirs:
+            os.chown(os.path.join(root, d), uid, gid)
+#            os.chmod(os.path.join(root, d), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+            os.chmod(os.path.join(root, d), 0o2770)
+
+        for f in files:
+            os.chown(os.path.join(root, f), uid, gid)
+#            os.chmod(os.path.join(root, f), stat.S_IRWXG | stat.S_ISGID | stat.S_IRWXU)
+            os.chmod(os.path.join(root, f), 0o2770)

build/jupyterhub/Dockerfile

@@ -0,0 +1,7 @@
+FROM jupyterhub/jupyterhub:latest
+MAINTAINER Tony Castronova <[email protected]>
+
+ADD install-deps.sh /tmp/install-deps.sh
+RUN sh /tmp/install-deps.sh
+
+ADD cull_idle_servers.py /srv/cull_idle_servers.py

build/jupyterhub/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash 
+
+
+docker build -t cuahsi/jupyterhub .
+