Updated App Setup File Renderer (#15)

* app store can parse toml files now

* compile setup.py to get correct variables

tethysapp/app_store/helpers.py

@@ -1,5 +1,7 @@
 import logging
 import os
+import re
+import toml
 
 from django.conf import settings
 from django.core.cache import cache
@@ -94,7 +96,7 @@ def apply_template(template_location, data, output_location):
         f.write(result)
 
 
-def parse_setup_py(file_location):
+def parse_setup_file(file_location):
     """Parses a setup.py file to get the app metadata
 
     Args:
@@ -103,29 +105,53 @@ def parse_setup_py(file_location):
     Returns:
         dict: A dictionary of key value pairs of application metadata
     """
-    params = {}
-    found_setup = False
-    with open(file_location, "r") as f:
-        for line in f.readlines():
-            if ("setup(" in line):
-                found_setup = True
-                continue
-            if found_setup:
-                if (")" in line):
-                    found_setup = False
-                    break
-                else:
-                    parts = line.split("=")
-                    if len(parts) < 2:
-                        continue
-                    value = parts[1].strip()
-                    if (value[-1] == ","):
-                        value = value[:-1]
-                    if (value[0] == "'" or value[0] == '"'):
-                        value = value[1:]
-                    if (value[-1] == "'" or value[-1] == '"'):
-                        value = value[:-1]
-                    params[parts[0].strip()] = value
+    if file_location.endswith("setup.py"):
+        import setuptools
+        setuptools.setup = lambda *a, **k: 0
+
+        params = {}
+        found_setup = False
+        with open(file_location, "r") as f:
+            for line in f.readlines():
+                if ("setup(" in line):
+                    found_setup = True
+                    continue
+                if found_setup:
+                    if (")" in line):
+                        found_setup = False
+                        break
+                    else:
+                        parts = line.split("=")
+                        if len(parts) < 2:
+                            continue
+                        value = parts[1].strip()
+                        if (value[-1] == ","):
+                            value = value[:-1]
+                        if (value[0] == "'" or value[0] == '"'):
+                            value = value[1:]
+                        if (value[-1] == "'" or value[-1] == '"'):
+                            value = value[:-1]
+                        params[parts[0].strip()] = value
+
+        with open(file_location, "r") as f:
+            c = f.read()
+
+        setup_helper_import = re.findall("(from .* import find_all_resource_files)", c)
+        if setup_helper_import:
+            c = c.replace(setup_helper_import[0], "from tethys_apps.app_installation import find_all_resource_files")
+
+        ns = {}
+        exec(compile(c, '__string__', 'exec'), {}, ns)
+        for key, value in params.items():
+            if value in ns:
+                params[key] = ns[value]
+    elif file_location.endswith(".toml"):
+        with open(file_location, 'r') as f:
+            config = toml.load(f)
+        params = config['project']
+    else:
+        raise Exception("A setup.py or .toml file must be provided")
+
     return params
 
 
@@ -165,19 +191,43 @@ def get_github_install_metadata(app_workspace):
             'installedVersion': '',
             'path': possible_app
         }
-        setup_path = os.path.join(possible_app, 'setup.py')
-        setup_py_data = parse_setup_py(setup_path)
-        installed_app["name"] = setup_py_data.get('name')
-        installed_app["installedVersion"] = setup_py_data.get('version')
-        installed_app["metadata"]["description"] = setup_py_data.get('description')
-        installed_app["author"] = setup_py_data.get('author')
-        installed_app["dev_url"] = setup_py_data.get('url')
+        setup_path = get_setup_path(possible_app)
+        setup_path_data = parse_setup_file(setup_path)
+        installed_app["name"] = setup_path_data.get('name')
+        installed_app["installedVersion"] = setup_path_data.get('version')
+        installed_app["metadata"]["description"] = setup_path_data.get('description')
+        installed_app["author"] = setup_path_data.get('author')
+        installed_app["dev_url"] = setup_path_data.get('url')
 
         github_installed_apps_list.append(installed_app)
     cache.set(CACHE_KEY, github_installed_apps_list)
     return github_installed_apps_list
 
 
+def get_setup_path(app_location):
+    """Returns a project file. Initially check for a setup.py file. Then check for a TOML file if a setup.py file was
+    not found. If neither of these files are found, raise an exception
+
+    Args:
+        app_location (_type_): _description_
+
+    Raises:
+        Exception: If a setup.py or toml file is not found, raise an exception
+
+    Returns:
+        str: Path to the project setup file, either a setup.py or a toml file
+    """
+    setup_path = os.path.join(app_location, 'setup.py')
+    if os.path.exists(setup_path):
+        return setup_path
+
+    for file in os.listdir(app_location):
+        if file.endswith("toml"):
+            return os.path.join(app_location, file)
+
+    raise Exception("Unable to find a project file for application")
+
+
 def get_conda_stores(active_only=False, conda_channels="all", sensitive_info=False):
     """Get the conda stores from the custom settings and decrypt tokens as well
 

tethysapp/app_store/submission_handlers.py

@@ -11,7 +11,7 @@
 from github.GithubException import UnknownObjectException, BadCredentialsException
 
 from pathlib import Path
-from .helpers import logger, send_notification, apply_template, parse_setup_py, get_conda_stores
+from .helpers import logger, send_notification, apply_template, parse_setup_file, get_setup_path, get_conda_stores
 
 CHANNEL_NAME = 'tethysapp'
 
@@ -213,16 +213,16 @@ def create_tethysapp_warehouse_release(repo, branch):
         repo.git.merge(branch)
 
 
-def generate_current_version(setup_py_data):
-    """Get the app version from the setup.py data
+def generate_current_version(setup_path_data):
+    """Get the app version from the setup file data
 
     Args:
-        setup_py_data (dict): App metadata from setup.py
+        setup_path_data (dict): App metadata from setup file
 
     Returns:
-        current_version (str): App version from the setup.py data
+        current_version (str): App version from the setup file data
     """
-    current_version = setup_py_data["version"]
+    current_version = setup_path_data["version"]
 
     return current_version
 
@@ -276,43 +276,43 @@ def create_upload_command(labels_string, source_files_path, recipe_path):
                    label, os.path.join(recipe_path, 'upload_command.txt'))
 
 
-def get_keywords_and_email(setup_py_data):
-    """Parses the setup.py dictionary to extract the keywords and the email
+def get_keywords_and_email(setup_path_data):
+    """Parses the setup file dictionary to extract the keywords and the email
 
     Args:
-        setup_py_data (dict): Application metadata derived from setup.py
+        setup_path_data (dict): Application metadata derived from setup file
 
     Returns:
         [keywords(list), email(str)]: A list of keywords and the author email
     """
-    keywords = setup_py_data.get("keywords")
+    keywords = setup_path_data.get("keywords")
     if keywords:
         keywords = keywords.replace(' ', '').replace('"', '').replace("'", '').split(',')
     else:
         keywords = []
-        logger.warning("No keywords found in setup.py")
+        logger.warning("No keywords found in setup file")
 
-    email = setup_py_data.get("author_email", "")
+    email = setup_path_data.get("author_email", "")
     if not email:
-        logger.warning("No author email found in setup.py")
+        logger.warning("No author email found in setup file")
 
     return keywords, email
 
 
-def create_template_data_for_install(app_github_dir, dev_url, setup_py_data):
+def create_template_data_for_install(app_github_dir, dev_url, setup_path_data):
     """Join the install_data information with the setup_py information to create template data for conda install
 
     Args:
         install_data (dict): Data from the application submission form by the user
-        setup_py_data (dict): Application metadata from the cloned repository's setup.py
+        setup_path_data (dict): Application metadata from the cloned repository's setup file
 
     Returns:
         dict: master dictionary use for templates, specifically for conda install
     """
     install_yml = os.path.join(app_github_dir, 'install.yml')
     with open(install_yml) as f:
         install_yml_file = yaml.safe_load(f)
-        metadata_dict = {**setup_py_data, "tethys_version": install_yml_file.get('tethys_version', '<=3.4.4'),
+        metadata_dict = {**setup_path_data, "tethys_version": install_yml_file.get('tethys_version', '<=3.4.4'),
                          "dev_url": dev_url}
 
     template_data = {
@@ -565,7 +565,7 @@ def process_branch(install_data, channel_layer, app_workspace):
     files_changed = False
     app_github_dir = get_gitsubmission_app_dir(app_workspace, app_name, conda_channel)
     repo = git.Repo(app_github_dir)
-    setup_py = os.path.join(app_github_dir, 'setup.py')
+    setup_path = get_setup_path(app_github_dir)
 
     # 2. Get sensitive information for store
     conda_store = get_conda_stores(conda_channels=conda_channel, sensitive_info=True)[0]
@@ -580,8 +580,8 @@ def process_branch(install_data, channel_layer, app_workspace):
     origin = repo.remote(name='origin')
     repo.git.checkout(branch)
     origin.pull()
-    setup_py_data = parse_setup_py(setup_py)
-    current_version = generate_current_version(setup_py_data)
+    setup_path_data = parse_setup_file(setup_path)
+    current_version = generate_current_version(setup_path_data)
 
     # 5. create head tethysapp_warehouse_release and checkout the head
     create_tethysapp_warehouse_release(repo, branch)
@@ -608,11 +608,11 @@ def process_branch(install_data, channel_layer, app_workspace):
     destination = os.path.join(recipe_path, 'meta.yaml')
     create_upload_command(labels_string, source_files_path, recipe_path)
 
-    # 10. Drop keywords from setup.py
-    keywords, email = get_keywords_and_email(setup_py_data)
+    # 10. Drop keywords from setup file
+    keywords, email = get_keywords_and_email(setup_path_data)
 
     # 11 get the data from the install.yml and create a metadata dict
-    template_data = create_template_data_for_install(app_github_dir, dev_url, setup_py_data)
+    template_data = create_template_data_for_install(app_github_dir, dev_url, setup_path_data)
     apply_template(source, template_data, destination)
     files_changed = copy_files_for_recipe(source, destination, files_changed)
 
@@ -622,7 +622,8 @@ def process_branch(install_data, channel_layer, app_workspace):
     files_changed = copy_files_for_recipe(source, destination, files_changed)
 
     # 13. Fix setup.py file to remove dependency on tethys
-    rel_package = fix_setup(setup_py)
+    if setup_path.endswith(".py"):
+        rel_package = fix_setup(setup_path)
 
     # 14. Update the dependencies of the package
     update_anaconda_dependencies(app_github_dir, recipe_path, source_files_path, keywords, email)

tethysapp/app_store/tests/files/pyproject.toml

@@ -0,0 +1,10 @@
+[project]
+name = "test_app"
+description = "example"
+long_description = "This is just an example for testing"
+license = "BSD-3"
+keywords = ["example", "test"]
+author = "Tester"
+author_email = "[email protected]"
+version = "0.0.1"
+url = ""

tethysapp/app_store/tests/unit_tests/test_helpers.py

@@ -1,9 +1,10 @@
 import pytest
 import shutil
+from pathlib import Path
 from unittest.mock import MagicMock
-from tethysapp.app_store.helpers import (parse_setup_py, get_conda_stores, check_all_present, run_process,
+from tethysapp.app_store.helpers import (parse_setup_file, get_conda_stores, check_all_present, run_process,
                                          send_notification, apply_template, get_github_install_metadata,
-                                         get_override_key, get_color_label_dict)
+                                         get_override_key, get_color_label_dict, get_setup_path)
 
 
 def test_get_override_key(mocker):
@@ -67,19 +68,41 @@ def test_apply_template(app_files_dir, tmp_path):
     assert output_location.read_text() == "anaconda upload --force --label main noarch/*.tar.bz2"
 
 
-def test_parse_setup_py(test_files_dir):
+def test_parse_setup_file_setup_py(test_files_dir):
     setup_py = test_files_dir / "setup.py"
 
-    parsed_data = parse_setup_py(setup_py)
+    parsed_data = parse_setup_file(str(setup_py))
 
     expected_data = {
-        'name': 'release_package', 'version': '0.0.1', 'description': 'example',
+        'name': 'tethysapp-test_app', 'version': '0.0.1', 'description': 'example',
         'long_description': 'This is just an example for testing', 'keywords': 'example,test',
         'author': 'Tester', 'author_email': '[email protected]', 'url': '', 'license': 'BSD-3'
     }
     assert parsed_data == expected_data
 
 
+def test_parse_setup_file_toml(test_files_dir):
+    project_toml = test_files_dir / "pyproject.toml"
+
+    parsed_data = parse_setup_file(str(project_toml))
+
+    expected_data = {
+        'name': 'test_app', 'version': '0.0.1', 'description': 'example',
+        'long_description': 'This is just an example for testing', 'keywords': ['example', 'test'],
+        'author': 'Tester', 'author_email': '[email protected]', 'url': '', 'license': 'BSD-3'
+    }
+    assert parsed_data == expected_data
+
+
+def test_parse_setup_file_bad_file(test_files_dir):
+    py_file = test_files_dir / "some_file.py"
+
+    with pytest.raises(Exception) as e:
+        parse_setup_file(str(py_file))
+
+    assert e.value.args[0] == 'A setup.py or .toml file must be provided'
+
+
 def test_get_github_install_metadata(tmp_path, test_files_dir, mocker):
     mock_cache = mocker.patch('tethysapp.app_store.helpers.cache')
     mock_cache.get.return_value = None
@@ -91,7 +114,7 @@ def test_get_github_install_metadata(tmp_path, test_files_dir, mocker):
     installed_apps = get_github_install_metadata(mock_workspace)
 
     expected_apps = {
-        'name': 'release_package', 'installed': True, 'installedVersion': '0.0.1',
+        'name': 'tethysapp-test_app', 'installed': True, 'installedVersion': '0.0.1',
         'metadata': {'channel': 'tethysapp', 'license': 'BSD 3-Clause License', 'description': 'example'},
         'path': str(mock_installed_app), 'author': 'Tester', 'dev_url': ''
     }
@@ -102,7 +125,7 @@ def test_get_github_install_metadata(tmp_path, test_files_dir, mocker):
 def test_get_github_install_metadata_cached(mocker):
     mock_cache = mocker.patch('tethysapp.app_store.helpers.cache')
     apps = [{
-        'name': 'release_package', 'installed': True, 'installedVersion': '0.0.1',
+        'name': 'tethysapp-test_app', 'installed': True, 'installedVersion': '0.0.1',
         'metadata': {'channel': 'tethysapp', 'license': 'BSD 3-Clause License', 'description': 'example'},
         'path': 'app_path', 'author': 'Tester', 'dev_url': ''
     }]
@@ -227,3 +250,28 @@ def test_get_color_label_dict(store):
     }]
     assert color_store_dict == expected_color_store_dict
     assert updated_stores == expected_updated_stores
+
+
+def test_get_setup_path_setup_py(tethysapp_base_with_application_files):
+    setup_path = get_setup_path(str(tethysapp_base_with_application_files))
+
+    assert Path(setup_path).is_file()
+    assert setup_path == str(tethysapp_base_with_application_files / "setup.py")
+
+
+def test_get_setup_path_toml(tmp_path, test_files_dir):
+    setup_helper = test_files_dir / "pyproject.toml"
+    tethysapp_setup_helper = tmp_path / "pyproject.toml"
+    shutil.copy(setup_helper, tethysapp_setup_helper)
+
+    setup_path = get_setup_path(str(tmp_path))
+
+    assert tethysapp_setup_helper.is_file()
+    assert setup_path == str(tethysapp_setup_helper)
+
+
+def test_get_setup_path_missing_file(tmp_path):
+    with pytest.raises(Exception) as e:
+        get_setup_path(str(tmp_path))
+
+    assert e.value.args[0] == 'Unable to find a project file for application'