From ee9cacff89cdd21ee797eff5b906cce664704ffd Mon Sep 17 00:00:00 2001 From: ckrew <153777116+ckrew@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:43:55 -0600 Subject: [PATCH] Updated App Setup File Renderer (#15) * app store can parse toml files now * compile setup.py to get correct variables --- tethysapp/app_store/helpers.py | 112 +++++++++++++----- tethysapp/app_store/submission_handlers.py | 47 ++++---- .../app_store/tests/files/pyproject.toml | 10 ++ .../tests/unit_tests/test_helpers.py | 62 ++++++++-- .../unit_tests/test_submission_handlers.py | 4 +- 5 files changed, 172 insertions(+), 63 deletions(-) create mode 100644 tethysapp/app_store/tests/files/pyproject.toml diff --git a/tethysapp/app_store/helpers.py b/tethysapp/app_store/helpers.py index f314105..1247900 100644 --- a/tethysapp/app_store/helpers.py +++ b/tethysapp/app_store/helpers.py @@ -1,5 +1,7 @@ import logging import os +import re +import toml from django.conf import settings from django.core.cache import cache @@ -94,7 +96,7 @@ def apply_template(template_location, data, output_location): f.write(result) -def parse_setup_py(file_location): +def parse_setup_file(file_location): """Parses a setup.py file to get the app metadata Args: @@ -103,29 +105,53 @@ def parse_setup_py(file_location): Returns: dict: A dictionary of key value pairs of application metadata """ - params = {} - found_setup = False - with open(file_location, "r") as f: - for line in f.readlines(): - if ("setup(" in line): - found_setup = True - continue - if found_setup: - if (")" in line): - found_setup = False - break - else: - parts = line.split("=") - if len(parts) < 2: - continue - value = parts[1].strip() - if (value[-1] == ","): - value = value[:-1] - if (value[0] == "'" or value[0] == '"'): - value = value[1:] - if (value[-1] == "'" or value[-1] == '"'): - value = value[:-1] - params[parts[0].strip()] = value + if file_location.endswith("setup.py"): + import setuptools + setuptools.setup = lambda *a, **k: 0 + + params = {} + found_setup = False + with open(file_location, "r") as f: + for line in f.readlines(): + if ("setup(" in line): + found_setup = True + continue + if found_setup: + if (")" in line): + found_setup = False + break + else: + parts = line.split("=") + if len(parts) < 2: + continue + value = parts[1].strip() + if (value[-1] == ","): + value = value[:-1] + if (value[0] == "'" or value[0] == '"'): + value = value[1:] + if (value[-1] == "'" or value[-1] == '"'): + value = value[:-1] + params[parts[0].strip()] = value + + with open(file_location, "r") as f: + c = f.read() + + setup_helper_import = re.findall("(from .* import find_all_resource_files)", c) + if setup_helper_import: + c = c.replace(setup_helper_import[0], "from tethys_apps.app_installation import find_all_resource_files") + + ns = {} + exec(compile(c, '__string__', 'exec'), {}, ns) + for key, value in params.items(): + if value in ns: + params[key] = ns[value] + elif file_location.endswith(".toml"): + with open(file_location, 'r') as f: + config = toml.load(f) + params = config['project'] + else: + raise Exception("A setup.py or .toml file must be provided") + return params @@ -165,19 +191,43 @@ def get_github_install_metadata(app_workspace): 'installedVersion': '', 'path': possible_app } - setup_path = os.path.join(possible_app, 'setup.py') - setup_py_data = parse_setup_py(setup_path) - installed_app["name"] = setup_py_data.get('name') - installed_app["installedVersion"] = setup_py_data.get('version') - installed_app["metadata"]["description"] = setup_py_data.get('description') - installed_app["author"] = setup_py_data.get('author') - installed_app["dev_url"] = setup_py_data.get('url') + setup_path = get_setup_path(possible_app) + setup_path_data = parse_setup_file(setup_path) + installed_app["name"] = setup_path_data.get('name') + installed_app["installedVersion"] = setup_path_data.get('version') + installed_app["metadata"]["description"] = setup_path_data.get('description') + installed_app["author"] = setup_path_data.get('author') + installed_app["dev_url"] = setup_path_data.get('url') github_installed_apps_list.append(installed_app) cache.set(CACHE_KEY, github_installed_apps_list) return github_installed_apps_list +def get_setup_path(app_location): + """Returns a project file. Initially check for a setup.py file. Then check for a TOML file if a setup.py file was + not found. If neither of these files are found, raise an exception + + Args: + app_location (_type_): _description_ + + Raises: + Exception: If a setup.py or toml file is not found, raise an exception + + Returns: + str: Path to the project setup file, either a setup.py or a toml file + """ + setup_path = os.path.join(app_location, 'setup.py') + if os.path.exists(setup_path): + return setup_path + + for file in os.listdir(app_location): + if file.endswith("toml"): + return os.path.join(app_location, file) + + raise Exception("Unable to find a project file for application") + + def get_conda_stores(active_only=False, conda_channels="all", sensitive_info=False): """Get the conda stores from the custom settings and decrypt tokens as well diff --git a/tethysapp/app_store/submission_handlers.py b/tethysapp/app_store/submission_handlers.py index 3c4c29d..39a94de 100644 --- a/tethysapp/app_store/submission_handlers.py +++ b/tethysapp/app_store/submission_handlers.py @@ -11,7 +11,7 @@ from github.GithubException import UnknownObjectException, BadCredentialsException from pathlib import Path -from .helpers import logger, send_notification, apply_template, parse_setup_py, get_conda_stores +from .helpers import logger, send_notification, apply_template, parse_setup_file, get_setup_path, get_conda_stores CHANNEL_NAME = 'tethysapp' @@ -213,16 +213,16 @@ def create_tethysapp_warehouse_release(repo, branch): repo.git.merge(branch) -def generate_current_version(setup_py_data): - """Get the app version from the setup.py data +def generate_current_version(setup_path_data): + """Get the app version from the setup file data Args: - setup_py_data (dict): App metadata from setup.py + setup_path_data (dict): App metadata from setup file Returns: - current_version (str): App version from the setup.py data + current_version (str): App version from the setup file data """ - current_version = setup_py_data["version"] + current_version = setup_path_data["version"] return current_version @@ -276,35 +276,35 @@ def create_upload_command(labels_string, source_files_path, recipe_path): label, os.path.join(recipe_path, 'upload_command.txt')) -def get_keywords_and_email(setup_py_data): - """Parses the setup.py dictionary to extract the keywords and the email +def get_keywords_and_email(setup_path_data): + """Parses the setup file dictionary to extract the keywords and the email Args: - setup_py_data (dict): Application metadata derived from setup.py + setup_path_data (dict): Application metadata derived from setup file Returns: [keywords(list), email(str)]: A list of keywords and the author email """ - keywords = setup_py_data.get("keywords") + keywords = setup_path_data.get("keywords") if keywords: keywords = keywords.replace(' ', '').replace('"', '').replace("'", '').split(',') else: keywords = [] - logger.warning("No keywords found in setup.py") + logger.warning("No keywords found in setup file") - email = setup_py_data.get("author_email", "") + email = setup_path_data.get("author_email", "") if not email: - logger.warning("No author email found in setup.py") + logger.warning("No author email found in setup file") return keywords, email -def create_template_data_for_install(app_github_dir, dev_url, setup_py_data): +def create_template_data_for_install(app_github_dir, dev_url, setup_path_data): """Join the install_data information with the setup_py information to create template data for conda install Args: install_data (dict): Data from the application submission form by the user - setup_py_data (dict): Application metadata from the cloned repository's setup.py + setup_path_data (dict): Application metadata from the cloned repository's setup file Returns: dict: master dictionary use for templates, specifically for conda install @@ -312,7 +312,7 @@ def create_template_data_for_install(app_github_dir, dev_url, setup_py_data): install_yml = os.path.join(app_github_dir, 'install.yml') with open(install_yml) as f: install_yml_file = yaml.safe_load(f) - metadata_dict = {**setup_py_data, "tethys_version": install_yml_file.get('tethys_version', '<=3.4.4'), + metadata_dict = {**setup_path_data, "tethys_version": install_yml_file.get('tethys_version', '<=3.4.4'), "dev_url": dev_url} template_data = { @@ -565,7 +565,7 @@ def process_branch(install_data, channel_layer, app_workspace): files_changed = False app_github_dir = get_gitsubmission_app_dir(app_workspace, app_name, conda_channel) repo = git.Repo(app_github_dir) - setup_py = os.path.join(app_github_dir, 'setup.py') + setup_path = get_setup_path(app_github_dir) # 2. Get sensitive information for store conda_store = get_conda_stores(conda_channels=conda_channel, sensitive_info=True)[0] @@ -580,8 +580,8 @@ def process_branch(install_data, channel_layer, app_workspace): origin = repo.remote(name='origin') repo.git.checkout(branch) origin.pull() - setup_py_data = parse_setup_py(setup_py) - current_version = generate_current_version(setup_py_data) + setup_path_data = parse_setup_file(setup_path) + current_version = generate_current_version(setup_path_data) # 5. create head tethysapp_warehouse_release and checkout the head create_tethysapp_warehouse_release(repo, branch) @@ -608,11 +608,11 @@ def process_branch(install_data, channel_layer, app_workspace): destination = os.path.join(recipe_path, 'meta.yaml') create_upload_command(labels_string, source_files_path, recipe_path) - # 10. Drop keywords from setup.py - keywords, email = get_keywords_and_email(setup_py_data) + # 10. Drop keywords from setup file + keywords, email = get_keywords_and_email(setup_path_data) # 11 get the data from the install.yml and create a metadata dict - template_data = create_template_data_for_install(app_github_dir, dev_url, setup_py_data) + template_data = create_template_data_for_install(app_github_dir, dev_url, setup_path_data) apply_template(source, template_data, destination) files_changed = copy_files_for_recipe(source, destination, files_changed) @@ -622,7 +622,8 @@ def process_branch(install_data, channel_layer, app_workspace): files_changed = copy_files_for_recipe(source, destination, files_changed) # 13. Fix setup.py file to remove dependency on tethys - rel_package = fix_setup(setup_py) + if setup_path.endswith(".py"): + rel_package = fix_setup(setup_path) # 14. Update the dependencies of the package update_anaconda_dependencies(app_github_dir, recipe_path, source_files_path, keywords, email) diff --git a/tethysapp/app_store/tests/files/pyproject.toml b/tethysapp/app_store/tests/files/pyproject.toml new file mode 100644 index 0000000..666a8e5 --- /dev/null +++ b/tethysapp/app_store/tests/files/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "test_app" +description = "example" +long_description = "This is just an example for testing" +license = "BSD-3" +keywords = ["example", "test"] +author = "Tester" +author_email = "tester@email.com" +version = "0.0.1" +url = "" \ No newline at end of file diff --git a/tethysapp/app_store/tests/unit_tests/test_helpers.py b/tethysapp/app_store/tests/unit_tests/test_helpers.py index 84cf9b3..62c33c0 100644 --- a/tethysapp/app_store/tests/unit_tests/test_helpers.py +++ b/tethysapp/app_store/tests/unit_tests/test_helpers.py @@ -1,9 +1,10 @@ import pytest import shutil +from pathlib import Path from unittest.mock import MagicMock -from tethysapp.app_store.helpers import (parse_setup_py, get_conda_stores, check_all_present, run_process, +from tethysapp.app_store.helpers import (parse_setup_file, get_conda_stores, check_all_present, run_process, send_notification, apply_template, get_github_install_metadata, - get_override_key, get_color_label_dict) + get_override_key, get_color_label_dict, get_setup_path) def test_get_override_key(mocker): @@ -67,19 +68,41 @@ def test_apply_template(app_files_dir, tmp_path): assert output_location.read_text() == "anaconda upload --force --label main noarch/*.tar.bz2" -def test_parse_setup_py(test_files_dir): +def test_parse_setup_file_setup_py(test_files_dir): setup_py = test_files_dir / "setup.py" - parsed_data = parse_setup_py(setup_py) + parsed_data = parse_setup_file(str(setup_py)) expected_data = { - 'name': 'release_package', 'version': '0.0.1', 'description': 'example', + 'name': 'tethysapp-test_app', 'version': '0.0.1', 'description': 'example', 'long_description': 'This is just an example for testing', 'keywords': 'example,test', 'author': 'Tester', 'author_email': 'tester@email.com', 'url': '', 'license': 'BSD-3' } assert parsed_data == expected_data +def test_parse_setup_file_toml(test_files_dir): + project_toml = test_files_dir / "pyproject.toml" + + parsed_data = parse_setup_file(str(project_toml)) + + expected_data = { + 'name': 'test_app', 'version': '0.0.1', 'description': 'example', + 'long_description': 'This is just an example for testing', 'keywords': ['example', 'test'], + 'author': 'Tester', 'author_email': 'tester@email.com', 'url': '', 'license': 'BSD-3' + } + assert parsed_data == expected_data + + +def test_parse_setup_file_bad_file(test_files_dir): + py_file = test_files_dir / "some_file.py" + + with pytest.raises(Exception) as e: + parse_setup_file(str(py_file)) + + assert e.value.args[0] == 'A setup.py or .toml file must be provided' + + def test_get_github_install_metadata(tmp_path, test_files_dir, mocker): mock_cache = mocker.patch('tethysapp.app_store.helpers.cache') mock_cache.get.return_value = None @@ -91,7 +114,7 @@ def test_get_github_install_metadata(tmp_path, test_files_dir, mocker): installed_apps = get_github_install_metadata(mock_workspace) expected_apps = { - 'name': 'release_package', 'installed': True, 'installedVersion': '0.0.1', + 'name': 'tethysapp-test_app', 'installed': True, 'installedVersion': '0.0.1', 'metadata': {'channel': 'tethysapp', 'license': 'BSD 3-Clause License', 'description': 'example'}, 'path': str(mock_installed_app), 'author': 'Tester', 'dev_url': '' } @@ -102,7 +125,7 @@ def test_get_github_install_metadata(tmp_path, test_files_dir, mocker): def test_get_github_install_metadata_cached(mocker): mock_cache = mocker.patch('tethysapp.app_store.helpers.cache') apps = [{ - 'name': 'release_package', 'installed': True, 'installedVersion': '0.0.1', + 'name': 'tethysapp-test_app', 'installed': True, 'installedVersion': '0.0.1', 'metadata': {'channel': 'tethysapp', 'license': 'BSD 3-Clause License', 'description': 'example'}, 'path': 'app_path', 'author': 'Tester', 'dev_url': '' }] @@ -227,3 +250,28 @@ def test_get_color_label_dict(store): }] assert color_store_dict == expected_color_store_dict assert updated_stores == expected_updated_stores + + +def test_get_setup_path_setup_py(tethysapp_base_with_application_files): + setup_path = get_setup_path(str(tethysapp_base_with_application_files)) + + assert Path(setup_path).is_file() + assert setup_path == str(tethysapp_base_with_application_files / "setup.py") + + +def test_get_setup_path_toml(tmp_path, test_files_dir): + setup_helper = test_files_dir / "pyproject.toml" + tethysapp_setup_helper = tmp_path / "pyproject.toml" + shutil.copy(setup_helper, tethysapp_setup_helper) + + setup_path = get_setup_path(str(tmp_path)) + + assert tethysapp_setup_helper.is_file() + assert setup_path == str(tethysapp_setup_helper) + + +def test_get_setup_path_missing_file(tmp_path): + with pytest.raises(Exception) as e: + get_setup_path(str(tmp_path)) + + assert e.value.args[0] == 'Unable to find a project file for application' diff --git a/tethysapp/app_store/tests/unit_tests/test_submission_handlers.py b/tethysapp/app_store/tests/unit_tests/test_submission_handlers.py index ba897a2..789068d 100644 --- a/tethysapp/app_store/tests/unit_tests/test_submission_handlers.py +++ b/tethysapp/app_store/tests/unit_tests/test_submission_handlers.py @@ -288,14 +288,14 @@ def test_create_template_data_for_install(complex_tethysapp): github_dir = complex_tethysapp dev_url = "https://github.com/notrealorg/fakeapp" setup_py_data = { - 'name': 'release_package', 'version': '0.0.1', 'description': 'example', + 'name': 'tethysapp-test_app', 'version': '0.0.1', 'description': 'example', 'long_description': 'This is just an example for testing', 'keywords': 'example,test', 'author': 'Tester', 'author_email': 'tester@email.com', 'url': '', 'license': 'BSD-3' } template_data = create_template_data_for_install(github_dir, dev_url, setup_py_data) expected_template_data = { - 'metadataObj': "{'name': 'release_package', 'version': '0.0.1', 'description': 'example', " + 'metadataObj': "{'name': 'tethysapp-test_app', 'version': '0.0.1', 'description': 'example', " "'long_description': 'This is just an example for testing', 'keywords': 'example,test', " "'author': 'Tester', 'author_email': 'tester@email.com', 'url': '', 'license': 'BSD-3', " "'tethys_version': '>=4.0', 'dev_url': 'https://github.com/notrealorg/fakeapp'}"