diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 4e1ef42..6982aa2 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -5,7 +5,7 @@ name: Upload Python Package on: release: - types: [created] + types: [published] jobs: deploy: @@ -24,6 +24,7 @@ jobs: pip install setuptools wheel twine - name: Build and publish env: + CIRRUS_VERSION: ${{ github.event.release.tag_name }} TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index c9a57ee..4b5a97b 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -30,6 +30,7 @@ jobs: pip install -r requirements-dev.txt # install black if available (Python 3.6 and above) pip install black || true + pip install . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/MANIFEST.in b/MANIFEST.in index 2c6e426..d5ad704 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include requirements.txt include requirements-dev.txt +graft src diff --git a/cirruslib/__init__.py b/cirruslib/__init__.py deleted file mode 100644 index 5a8fc23..0000000 --- a/cirruslib/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .catalog import Catalog, Catalogs -from .logging import get_task_logger -from .statedb import StateDB, STATES -from .version import __version__ \ No newline at end of file diff --git a/cirruslib/version.py b/cirruslib/version.py deleted file mode 100644 index 93b60a1..0000000 --- a/cirruslib/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '0.5.1' diff --git a/setup.py b/setup.py index bd9ec41..033e06d 100755 --- a/setup.py +++ b/setup.py @@ -1,24 +1,72 @@ #!/usr/bin/env python -from setuptools import setup, find_packages -from imp import load_source -from os import path -import io +import os +import os.path +import subprocess -__version__ = load_source('cirruslib.version', 'cirruslib/version.py').__version__ +from setuptools import setup, find_namespace_packages -here = path.abspath(path.dirname(__file__)) -# get the dependencies and installs -with io.open(path.join(here, 'requirements.txt'), encoding='utf-8') as f: - all_reqs = f.read().split('\n') +HERE = os.path.abspath(os.path.dirname(__file__)) + + +# gets the version from the latest tag via git describe +# so we don't have to do anything to manage version number +# aside from tagging releases +def git_version(gitdir, default='0.0.0'): + try: + desc = subprocess.run( + [ + 'git', + '--git-dir', + gitdir, + 'describe', + '--long', + '--tags', + '--dirty', + ], + capture_output=True, + ) + except Exception: + return default + + if desc.returncode != 0: + return default + + # example output: v0.5.1-8-gb38722d-dirty + # parts are: + # 0 - last tag + # 1 - commits since last tag (0 if same commit as tag) + # 2 - short hash of current commit + # 3 - dirty (if repo state is dirty) + parts = desc.stdout.decode().strip().lstrip('v').split('-', maxsplit=2) + if int(parts[1]) > 0 or 'dirty' in parts[2]: + return f'{parts[0]}+{parts[1]}.{parts[2].replace("-",".")}' + else: + return parts[0] + + +# in the case of a tagged release, we +# are passed a version in an env var +VERSION = os.environ.get( + 'CIRRUS_VERSION', + git_version(os.path.join(HERE, '.git')), +) + + +with open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f: + readme = f.read() + +with open(os.path.join(HERE, 'requirements.txt'), encoding='utf-8') as f: + reqs = f.read().split('\n') + +install_requires = [x.strip() for x in reqs if 'git+' not in x] +dependency_links = [x.strip().replace('git+', '') for x in reqs if 'git+' not in x] -install_requires = [x.strip() for x in all_reqs if 'git+' not in x] -dependency_links = [x.strip().replace('git+', '') for x in all_reqs if 'git+' not in x] setup( name='cirrus-lib', author='Matthew Hanson (matthewhanson), Element 84', - version=__version__, + version=VERSION, description='Cirrus Library', url='https://github.com/cirrus-geo/cirrus-lib.git', license='Apache-2.0', @@ -29,7 +77,8 @@ 'Programming Language :: Python :: 3.8' ], keywords='', - packages=find_packages(exclude=['docs', 'test*']), + packages=find_namespace_packages('src'), + package_dir={'': 'src'}, include_package_data=True, install_requires=install_requires, dependency_links=dependency_links, diff --git a/src/cirrus/lib/__init__.py b/src/cirrus/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cirruslib/catalog.py b/src/cirrus/lib/catalog.py similarity index 97% rename from cirruslib/catalog.py rename to src/cirrus/lib/catalog.py index 69c79d6..4076323 100644 --- a/cirruslib/catalog.py +++ b/src/cirrus/lib/catalog.py @@ -10,10 +10,10 @@ from typing import Dict, Optional, List from boto3utils import s3 -from cirruslib.statedb import StateDB -from cirruslib.logging import get_task_logger -from cirruslib.transfer import get_s3_session -from cirruslib.utils import get_path, property_match +from cirrus.lib.statedb import StateDB +from cirrus.lib.logging import get_task_logger +from cirrus.lib.transfer import get_s3_session +from cirrus.lib.utils import get_path, property_match # envvars CATALOG_BUCKET = os.getenv('CIRRUS_CATALOG_BUCKET', None) @@ -62,7 +62,7 @@ def __init__(self, *args, update=False, state_item=None, **kwargs): assert(len(self['features']) > 0) for item in self['features']: if 'links' not in item: - item['links'] = [] + item['links'] = [] # update collection IDs of member Items self.assign_collections() @@ -101,14 +101,14 @@ def from_payload(cls, payload: Dict, **kwargs) -> Catalog: def update(self): if 'collections' in self['process']: - # allow overriding of collections name + # allow overriding of collections name collections_str = self['process']['collections'] else: # otherwise, get from items cols = sorted(list(set([i['collection'] for i in self['features'] if 'collection' in i]))) input_collections = cols if len(cols) != 0 else 'none' collections_str = '/'.join(input_collections) - + items_str = '/'.join(sorted(list([i['id'] for i in self['features']]))) if 'id' not in self: self['id'] = f"{collections_str}/workflow-{self['process']['workflow']}/{items_str}" @@ -218,7 +218,7 @@ def publish_to_s3(self, bucket, public=False) -> List: # publish to bucket headers = opts.get('headers', {}) - + extra = {'ContentType': 'application/json'} extra.update(headers) s3session.upload_json(item, url, public=public, extra=extra) @@ -261,7 +261,7 @@ def sns_attributes(self, item) -> Dict: 'bbox.ur_lat': { 'DataType': 'Number', 'StringValue': str(item['bbox'][3]) - } + } } if 'eo:cloud_cover' in item['properties']: attr['cloud_cover'] = { @@ -288,7 +288,7 @@ def publish_to_sns(self, topic_arn=PUBLISH_TOPIC_ARN): """ for item in self['features']: response = snsclient.publish(TopicArn=topic_arn, Message=json.dumps(item), - MessageAttributes=self.sns_attributes(item)) + MessageAttributes=self.sns_attributes(item)) self.logger.debug(f"Published item to {topic_arn}") def process(self) -> str: @@ -425,7 +425,7 @@ def process(self, replace=False): catalog (Dict): A Cirrus Input Catalog """ catids = [] - # check existing states + # check existing states states = self.get_states() for cat in self.catalogs: _replace = replace or cat['process'].get('replace', False) diff --git a/cirruslib/errors.py b/src/cirrus/lib/errors.py similarity index 94% rename from cirruslib/errors.py rename to src/cirrus/lib/errors.py index 55e4a9f..53aefca 100644 --- a/cirruslib/errors.py +++ b/src/cirrus/lib/errors.py @@ -6,4 +6,4 @@ class InvalidInput(Exception): Args: Exception (Exception): Base class """ - pass \ No newline at end of file + pass diff --git a/cirruslib/logging.py b/src/cirrus/lib/logging.py similarity index 98% rename from cirruslib/logging.py rename to src/cirrus/lib/logging.py index 784c9ef..0b69886 100644 --- a/cirruslib/logging.py +++ b/src/cirrus/lib/logging.py @@ -39,7 +39,7 @@ "handlers": ["standard"], "level": getenv('CIRRUS_LOG_LEVEL', 'DEBUG') }, - "cirruslib": { + "cirrus.lib": { "handlers": ["standard"], "level": getenv('CIRRUS_LOG_LEVEL', 'DEBUG') } diff --git a/cirruslib/statedb.py b/src/cirrus/lib/statedb.py similarity index 100% rename from cirruslib/statedb.py rename to src/cirrus/lib/statedb.py diff --git a/cirruslib/transfer.py b/src/cirrus/lib/transfer.py similarity index 99% rename from cirruslib/transfer.py rename to src/cirrus/lib/transfer.py index e0d7483..daf953c 100644 --- a/cirruslib/transfer.py +++ b/src/cirrus/lib/transfer.py @@ -8,7 +8,7 @@ from copy import deepcopy from dateutil.parser import parse as dateparse from os import getenv, path as op -from cirruslib.utils import get_path +from cirrus.lib.utils import get_path from typing import Dict, Optional, List diff --git a/cirruslib/utils.py b/src/cirrus/lib/utils.py similarity index 100% rename from cirruslib/utils.py rename to src/cirrus/lib/utils.py diff --git a/src/cirruslib/__init__.py b/src/cirruslib/__init__.py new file mode 100644 index 0000000..142948f --- /dev/null +++ b/src/cirruslib/__init__.py @@ -0,0 +1,7 @@ +import warnings + +warnings.warn( + "Imports via 'cirruslib' will be removed in a future cirrus version. " + "Update all imports to instead use 'cirrus.lib'.", + DeprecationWarning, +) diff --git a/src/cirruslib/catalog.py b/src/cirruslib/catalog.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/catalog.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/src/cirruslib/errors.py b/src/cirruslib/errors.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/errors.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/src/cirruslib/logging.py b/src/cirruslib/logging.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/logging.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/src/cirruslib/statedb.py b/src/cirruslib/statedb.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/statedb.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/src/cirruslib/transfer.py b/src/cirruslib/transfer.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/transfer.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/src/cirruslib/utils.py b/src/cirruslib/utils.py new file mode 100644 index 0000000..7fe10e0 --- /dev/null +++ b/src/cirruslib/utils.py @@ -0,0 +1,4 @@ +import importlib +import sys +module = __name__.split('.')[-1] +sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}') diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 495b5fe..2b2d204 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -3,7 +3,7 @@ import json import unittest -from cirruslib import Catalog +from cirrus.lib.catalog import Catalog testpath = os.path.dirname(__file__) diff --git a/tests/test_statedb.py b/tests/test_statedb.py index 56e231a..5f5b9f2 100644 --- a/tests/test_statedb.py +++ b/tests/test_statedb.py @@ -10,7 +10,7 @@ from copy import deepcopy from datetime import datetime from decimal import Decimal -from cirruslib.statedb import StateDB, STATES +from cirrus.lib.statedb import StateDB, STATES ## fixtures testpath = os.path.dirname(__file__) diff --git a/tests/test_transfer.py b/tests/test_transfer.py index 6dd3366..9f8a088 100644 --- a/tests/test_transfer.py +++ b/tests/test_transfer.py @@ -5,7 +5,7 @@ import unittest from boto3utils import s3 -from cirruslib import transfer +from cirrus.lib import transfer from shutil import rmtree testpath = f"{os.path.dirname(__file__)}/test_transfer"