Added Tests 93% Python Coverage (#12)

* added function docstrings to tested functions

* updated some function names for clarity

* updated get_head_and_tag_names to use remote repo instead of local

* updated get_workflow_job_url to use all python apis instead of request library

* updated some variable names to be more clear

* created helper functions to remove duplicated code

* move some helper functions into resource helpers

* linted all code with flake8

* finished tests for most python files

* fixed minor mamba command issue

tethysapp/app_store/tests/.coveragerc → .coveragerc

@@ -6,4 +6,5 @@ omit =
    *tests/*,
    *urls.py,
    *wsgi.py,
-   manage.py
+   manage.py,
+   *workspaces/*

.flake8

@@ -1,2 +1,4 @@
 [flake8]
 max-line-length = 120
+exclude = tethysapp/app_store/workspaces/
+

.gitignore

@@ -15,6 +15,7 @@ tethysapp/app_store/workspaces/app_workspace/gitsubmission/
 tethysapp/app_store/workspaces/app_workspace/install_status/
 tethysapp/app_store/workspaces/app_workspace/logs/
 tethysapp/app_store/workspaces/app_workspace/develop/
+tethysapp/app_store/model.py
 
 docs/_build
 .vscode/

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+DJANGO_SETTINGS_MODULE=tethys_portal.settings
+python_files = test_*.py *_tests.py
+addopts = --ignore-glob=*workspaces/* --cov --cov-report term-missing --disable-warnings

tethysapp/app_store/begin_install.py

@@ -2,80 +2,79 @@
 import time
 import importlib
 import subprocess
+import tethysapp
+import site
 
 from django.core.cache import cache
 
 from subprocess import call
 
-from .helpers import check_all_present, get_app_instance_from_path, logger, send_notification
-from .resource_helpers import get_resource
+from .helpers import check_all_present, logger, send_notification
+from .resource_helpers import get_resource, get_app_instance_from_path
 
 
 def handle_property_not_present(prop):
+    """Handles any issues if certain properties/metadata are not present
+
+    Args:
+        prop (dict): application metadata
+    """
     # TODO: Generate an error message that metadata is incorrect for this application
     pass
 
 
-def process_post_install_scripts(path):
-    # Check if scripts directory exists
-    scripts_dir = os.path.join(path, 'scripts')
+def process_post_install_scripts(scripts_dir):
+    """Process any post installation scripts from the installed application
+
+    Args:
+        path (str): Path to the application base directory
+    """
     if os.path.exists(scripts_dir):
-        logger.info("TODO: Process scripts dir.")
         # Currently only processing the pip install script, but need to add ability to process post scripts as well
+        pass
 
 
 def detect_app_dependencies(app_name, channel_layer, notification_method=send_notification):
-    """
-    Method goes through the app.py and determines the following:
-    1.) Any services required
-    2.) Thredds?
-    3.) Geoserver Requirement?
-    4.) Custom Settings required for installation?
-    """
+    """Check the application for pip (via a pip_install.sh) and custom setting dependencies
 
-    # Get Conda Packages location
-    # Tried using conda_prefix from env as well as conda_info but both of them are not reliable
-    # Best method is to import the module and try and get the location from that path
-    # @TODO : Ensure that this works through multiple runs
-    import tethysapp
-    # store_pkg = importlib.import_module(app_channel)
+    Args:
+        app_name (str): Name of the application being installed
+        channel_layer (Django Channels Layer): Asynchronous Django channel layer from the websocket consumer
+        notification_method (Object, optional): Method of how to send notifications. Defaults to send_notification
+            which is a WebSocket.
+    """
 
+    logger.info("Running a DB sync")
     call(['tethys', 'db', 'sync'])
     cache.clear()
-    # clear_url_caches()
+
     # After install we need to update the sys.path variable so we can see the new apps that are installed.
     # We need to do a reload here of the sys.path and then reload the tethysapp
     # https://stackoverflow.com/questions/25384922/how-to-refresh-sys-path
-    import site
     importlib.reload(site)
     importlib.reload(tethysapp)
-    # importlib.reload(store_pkg)
 
-    # paths = list()
-    # paths = list(filter(lambda x: app_name in x, store_pkg.__path__))
-    paths = list(filter(lambda x: app_name in x, tethysapp.__path__))
+    installed_app_paths = [path for path in tethysapp.__path__ if app_name in path]
 
-    if len(paths) < 1:
+    if len(installed_app_paths) < 1:
         logger.error("Can't find the installed app location.")
         return
 
-    # Check for any pre install script to install pip dependencies
-
-    app_folders = next(os.walk(paths[0]))[1]
-    app_scripts_path = os.path.join(paths[0], app_folders[0], 'scripts')
+    installed_app_path = installed_app_paths[0]
+    app_folders = next(os.walk(installed_app_path))[1]
+    app_scripts_path = os.path.join(installed_app_path, app_folders[0], 'scripts')
     pip_install_script_path = os.path.join(app_scripts_path, 'install_pip.sh')
 
     if os.path.exists(pip_install_script_path):
         logger.info("PIP dependencies found. Running Pip install script")
 
         notification_method("Running PIP install....", channel_layer)
-        p = subprocess.Popen(['sh', pip_install_script_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        process = subprocess.Popen(['sh', pip_install_script_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         while True:
-            output = p.stdout.readline()
+            output = process.stdout.readline()
             if output == '':
                 break
             if output:
-                # Checkpoints for the output
                 str_output = str(output.strip())
                 logger.info(str_output)
                 if (check_all_present(str_output, ['PIP Install Complete'])):
@@ -84,8 +83,9 @@ def detect_app_dependencies(app_name, channel_layer, notification_method=send_no
         notification_method("PIP install completed", channel_layer)
 
     # @TODO: Add support for post installation scripts as well.
+    process_post_install_scripts(app_scripts_path)
 
-    app_instance = get_app_instance_from_path(paths)
+    app_instance = get_app_instance_from_path(installed_app_paths)
     custom_settings_json = []
     custom_settings = app_instance.custom_settings()
 
@@ -102,15 +102,23 @@ def detect_app_dependencies(app_name, channel_layer, notification_method=send_no
         "data": custom_settings_json,
         "returnMethod": "set_custom_settings",
         "jsHelperFunction": "processCustomSettings",
-        "app_py_path": str(paths[0])
+        "app_py_path": str(installed_app_path)
     }
     notification_method(get_data_json, channel_layer)
 
     return
 
 
-def conda_install(app_metadata, app_channel, app_label, app_version, channel_layer):
+def mamba_install(app_metadata, app_channel, app_label, app_version, channel_layer):
+    """Run a conda install with a application using the anaconda package
 
+    Args:
+        app_metadata (dict): Dictionary representing an app and its conda metadata
+        app_channel (str): Conda channel to use for the app install
+        app_label (str): Conda label to use for the app install
+        app_version (str): App version to use for app install
+        channel_layer (Django Channels Layer): Asynchronous Django channel layer from the websocket consumer
+    """
     start_time = time.time()
     send_notification("Mamba install may take a couple minutes to complete depending on how complicated the "
                       "environment is. Please wait....", channel_layer)
@@ -121,7 +129,7 @@ def conda_install(app_metadata, app_channel, app_label, app_version, channel_lay
 
     # Running the conda install as a subprocess to get more visibility into the running process
     dir_path = os.path.dirname(os.path.realpath(__file__))
-    script_path = os.path.join(dir_path, "scripts", "conda_install.sh")
+    script_path = os.path.join(dir_path, "scripts", "mamba_install.sh")
 
     app_name = app_metadata['name'] + "=" + app_version
 
@@ -133,9 +141,8 @@ def conda_install(app_metadata, app_channel, app_label, app_version, channel_lay
     install_command = [script_path, app_name, label_channel]
 
     # Running this sub process, in case the library isn't installed, triggers a restart.
-
     p = subprocess.Popen(install_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
+    success = True
     while True:
         output = p.stdout.readline()
         if output == '':
@@ -153,31 +160,46 @@ def conda_install(app_metadata, app_channel, app_label, app_version, channel_lay
             if (check_all_present(str_output, ['All requested packages already installed.'])):
                 send_notification("Application package is already installed in this conda environment.",
                                   channel_layer)
-            if (check_all_present(str_output, ['Mamba Install Complete'])):
-                break
+            if (check_all_present(str_output, ['libmamba Could not solve for environment specs', 'critical'])):
+                success = False
+                send_notification("Failed to resolve environment specs when installing.",
+                                  channel_layer)
             if (check_all_present(str_output, ['Found conflicts!'])):
-                send_notification("Mamba install found conflicts."
-                                  "Please try running the following command in your terminal's"
+                success = False
+                send_notification("Mamba install found conflicts. "
+                                  "Please try running the following command in your terminal's "
                                   "conda environment to attempt a manual installation : "
-                                  "mamba install -c " + label_channel + " " + app_name,
+                                  f"mamba install -c {label_channel} {app_name}",
                                   channel_layer)
+            if (check_all_present(str_output, ['Mamba Install Complete'])):
+                break
 
     send_notification("Mamba install completed in %.2f seconds." % (time.time() - start_time), channel_layer)
 
+    return success
+
 
 def begin_install(installData, channel_layer, app_workspace):
+    """Using the install data, this function will retrieve a specific app resource and install the application as well
+    as update any app dependencies
 
+    Args:
+        installData (dict): User provided information about the application that should be installed
+        channel_layer (Django Channels Layer): Asynchronous Django channel layer from the websocket consumer
+        app_workspace (str): Path pointing to the app workspace within the app store
+    """
     resource = get_resource(installData["name"], installData['channel'], installData['label'], app_workspace)
+    if not resource:
+        send_notification(f"Failed to get the {installData['name']} resource", channel_layer)
+        return
 
-    send_notification("Starting installation of app: " + resource['name'] + " from store " + installData['channel'] +
-                      " with label " + installData['label'], channel_layer)
-    send_notification("Installing Version: " + installData["version"], channel_layer)
+    send_notification(f"Starting installation of app: {resource['name']} from store {installData['channel']} "
+                      f"with label {installData['label']}", channel_layer)
+    send_notification(f"Installing Version: {installData['version']}", channel_layer)
 
-    try:
-        conda_install(resource, installData['channel'], installData['label'], installData["version"], channel_layer)
-    except Exception as e:
-        logger.error("Error while running conda install")
-        logger.error(e)
+    successful_install = mamba_install(resource, installData['channel'], installData['label'], installData["version"],
+                                       channel_layer)
+    if not successful_install:
         send_notification("Error while Installing Conda package. Please check logs for details", channel_layer)
         return
 

tethysapp/app_store/controllers.py

@@ -5,28 +5,30 @@
 from tethys_sdk.routing import controller
 
 from .resource_helpers import get_stores_reformatted
-
-from .app import AppStore as app
-from .utilities import decrypt
+from .helpers import get_conda_stores
 ALL_RESOURCES = []
 CACHE_KEY = "warehouse_app_resources"
 
 
 @controller(
     name='home',
     url='app-store',
-    permissions_required='use_app_store',
-    app_workspace=True,
+    permissions_required='use_app_store'
 )
-def home(request, app_workspace):
-    available_stores_data_dict = app.get_custom_setting("stores_settings")['stores']
-    encryption_key = app.get_custom_setting("encryption_key")
-    for store in available_stores_data_dict:
-        store['github_token'] = decrypt(store['github_token'], encryption_key)
+def home(request):
+    """Created the context for the home page of the app store
+
+    Args:
+        request (Django Request): Django request object containing information about the user and user request
+
+    Returns:
+        object: Rendered html Django object
+    """
+    available_stores = get_conda_stores()
 
     context = {
-        'storesData': available_stores_data_dict,
-        'show_stores': True if len(available_stores_data_dict) > 0 else False
+        'storesData': available_stores,
+        'show_stores': True if len(available_stores) > 0 else False
     }
 
     return render(request, 'app_store/home.html', context)
@@ -35,17 +37,20 @@ def home(request, app_workspace):
 @controller(
     name='get_available_stores',
     url='app-store/get_available_stores',
-    permissions_required='use_app_store',
-    app_workspace=True,
+    permissions_required='use_app_store'
 )
-def get_available_stores(request, app_workspace):
+def get_available_stores(request):
+    """Retrieves the available stores through an ajax request
 
-    available_stores_data_dict = app.get_custom_setting("stores_settings")
-    encryption_key = app.get_custom_setting("encryption_key")
-    for store in available_stores_data_dict['stores']:
-        store['github_token'] = decrypt(store['github_token'], encryption_key)
+    Args:
+        request (Django Request): Django request object containing information about the user and user request
 
-    return JsonResponse(available_stores_data_dict)
+    Returns:
+        JsonResponse: A json reponse of the available conda stores
+    """
+    available_stores = get_conda_stores()
+    available_stores_dict = {"stores": available_stores}
+    return JsonResponse(available_stores_dict)
 
 
 @controller(
@@ -54,12 +59,12 @@ def get_available_stores(request, app_workspace):
     permissions_required='use_app_store',
     app_workspace=True,
 )
-def get_resources_multiple_stores(request, app_workspace):
+def get_merged_resources(request, app_workspace):
 
     stores_active = request.GET.get('active_store')
 
     object_stores_formatted_by_label_and_channel = get_stores_reformatted(app_workspace, refresh=False,
-                                                                          stores=stores_active)
+                                                                          conda_channels=stores_active)
 
     tethys_version_regex = re.search(r'([\d.]+[\d])', tethys_version).group(1)
     object_stores_formatted_by_label_and_channel['tethysVersion'] = tethys_version_regex

tethysapp/app_store/git_install_handlers.py

@@ -22,8 +22,9 @@
 from subprocess import (Popen, PIPE, STDOUT)
 from datetime import datetime
 
+from conda.cli.python_api import run_command as conda_run, Commands
 from .app import AppStore as app
-from .helpers import Commands, conda_run, get_override_key, logger
+from .helpers import get_override_key, logger
 from .installation_handlers import restart_server
 
 FNULL = open(os.devnull, 'w')