Merge pull request #31 from cirrus-geo/reorg

Reorganization and tag versioning

.github/workflows/python-publish.yml

@@ -5,7 +5,7 @@ name: Upload Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 jobs:
   deploy:
@@ -24,6 +24,7 @@ jobs:
         pip install setuptools wheel twine
     - name: Build and publish
       env:
+        CIRRUS_VERSION: ${{ github.event.release.tag_name }}
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |

.github/workflows/python-test.yml

@@ -30,6 +30,7 @@ jobs:
           pip install -r requirements-dev.txt
           # install black if available (Python 3.6 and above)
           pip install black || true
+          pip install .
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names

MANIFEST.in

@@ -1,2 +1,3 @@
 include requirements.txt
 include requirements-dev.txt
+graft src

setup.py

@@ -1,24 +1,72 @@
 #!/usr/bin/env python
-from setuptools import setup, find_packages
-from imp import load_source
-from os import path
-import io
+import os
+import os.path
+import subprocess
 
-__version__ = load_source('cirruslib.version', 'cirruslib/version.py').__version__
+from setuptools import setup, find_namespace_packages
 
-here = path.abspath(path.dirname(__file__))
 
-# get the dependencies and installs
-with io.open(path.join(here, 'requirements.txt'), encoding='utf-8') as f:
-    all_reqs = f.read().split('\n')
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+
+# gets the version from the latest tag via git describe
+# so we don't have to do anything to manage version number
+# aside from tagging releases
+def git_version(gitdir, default='0.0.0'):
+    try:
+        desc = subprocess.run(
+            [
+                'git',
+                '--git-dir',
+                gitdir,
+                'describe',
+                '--long',
+                '--tags',
+                '--dirty',
+            ],
+            capture_output=True,
+        )
+    except Exception:
+        return default
+
+    if desc.returncode != 0:
+        return default
+
+    # example output: v0.5.1-8-gb38722d-dirty
+    # parts are:
+    #  0 - last tag
+    #  1 - commits since last tag (0 if same commit as tag)
+    #  2 - short hash of current commit
+    #  3 - dirty (if repo state is dirty)
+    parts = desc.stdout.decode().strip().lstrip('v').split('-', maxsplit=2)
+    if int(parts[1]) > 0 or 'dirty' in parts[2]:
+        return f'{parts[0]}+{parts[1]}.{parts[2].replace("-",".")}'
+    else:
+        return parts[0]
+
+
+# in the case of a tagged release, we
+# are passed a version in an env var
+VERSION = os.environ.get(
+    'CIRRUS_VERSION',
+    git_version(os.path.join(HERE, '.git')),
+)
+
+
+with open(os.path.join(HERE, 'README.md'), encoding='utf-8') as f:
+    readme = f.read()
+
+with open(os.path.join(HERE, 'requirements.txt'), encoding='utf-8') as f:
+    reqs = f.read().split('\n')
+
+install_requires = [x.strip() for x in reqs if 'git+' not in x]
+dependency_links = [x.strip().replace('git+', '') for x in reqs if 'git+' not in x]
 
-install_requires = [x.strip() for x in all_reqs if 'git+' not in x]
-dependency_links = [x.strip().replace('git+', '') for x in all_reqs if 'git+' not in x]
 
 setup(
     name='cirrus-lib',
     author='Matthew Hanson (matthewhanson), Element 84',
-    version=__version__,
+    version=VERSION,
     description='Cirrus Library',
     url='https://github.com/cirrus-geo/cirrus-lib.git',
     license='Apache-2.0',
@@ -29,7 +77,8 @@
         'Programming Language :: Python :: 3.8'
     ],
     keywords='',
-    packages=find_packages(exclude=['docs', 'test*']),
+    packages=find_namespace_packages('src'),
+    package_dir={'': 'src'},
     include_package_data=True,
     install_requires=install_requires,
     dependency_links=dependency_links,

cirruslib/catalog.py → src/cirrus/lib/catalog.py

@@ -10,10 +10,10 @@
 from typing import Dict, Optional, List
 
 from boto3utils import s3
-from cirruslib.statedb import StateDB
-from cirruslib.logging import get_task_logger
-from cirruslib.transfer import get_s3_session
-from cirruslib.utils import get_path, property_match
+from cirrus.lib.statedb import StateDB
+from cirrus.lib.logging import get_task_logger
+from cirrus.lib.transfer import get_s3_session
+from cirrus.lib.utils import get_path, property_match
 
 # envvars
 CATALOG_BUCKET = os.getenv('CIRRUS_CATALOG_BUCKET', None)
@@ -62,7 +62,7 @@ def __init__(self, *args, update=False, state_item=None, **kwargs):
         assert(len(self['features']) > 0)
         for item in self['features']:
             if 'links' not in item:
-                item['links'] = [] 
+                item['links'] = []
 
         # update collection IDs of member Items
         self.assign_collections()
@@ -101,14 +101,14 @@ def from_payload(cls, payload: Dict, **kwargs) -> Catalog:
 
     def update(self):
         if 'collections' in self['process']:
-            # allow overriding of collections name 
+            # allow overriding of collections name
             collections_str = self['process']['collections']
         else:
             # otherwise, get from items
             cols = sorted(list(set([i['collection'] for i in self['features'] if 'collection' in i])))
             input_collections = cols if len(cols) != 0 else 'none'
             collections_str = '/'.join(input_collections)
-        
+
         items_str = '/'.join(sorted(list([i['id'] for i in self['features']])))
         if 'id' not in self:
             self['id'] = f"{collections_str}/workflow-{self['process']['workflow']}/{items_str}"
@@ -218,7 +218,7 @@ def publish_to_s3(self, bucket, public=False) -> List:
 
             # publish to bucket
             headers = opts.get('headers', {})
-            
+
             extra = {'ContentType': 'application/json'}
             extra.update(headers)
             s3session.upload_json(item, url, public=public, extra=extra)
@@ -261,7 +261,7 @@ def sns_attributes(self, item) -> Dict:
             'bbox.ur_lat': {
                 'DataType': 'Number',
                 'StringValue': str(item['bbox'][3])
-            }     
+            }
         }
         if 'eo:cloud_cover' in item['properties']:
             attr['cloud_cover'] = {
@@ -288,7 +288,7 @@ def publish_to_sns(self, topic_arn=PUBLISH_TOPIC_ARN):
         """
         for item in self['features']:
             response = snsclient.publish(TopicArn=topic_arn, Message=json.dumps(item),
-                                        MessageAttributes=self.sns_attributes(item))         
+                                        MessageAttributes=self.sns_attributes(item))
             self.logger.debug(f"Published item to {topic_arn}")
 
     def process(self) -> str:
@@ -425,7 +425,7 @@ def process(self, replace=False):
             catalog (Dict): A Cirrus Input Catalog
         """
         catids = []
-        # check existing states            
+        # check existing states
         states = self.get_states()
         for cat in self.catalogs:
             _replace = replace or cat['process'].get('replace', False)

cirruslib/errors.py → src/cirrus/lib/errors.py

@@ -6,4 +6,4 @@ class InvalidInput(Exception):
     Args:
         Exception (Exception): Base class
     """
-    pass
+    pass

cirruslib/logging.py → src/cirrus/lib/logging.py

@@ -39,7 +39,7 @@
           "handlers": ["standard"],
           "level": getenv('CIRRUS_LOG_LEVEL', 'DEBUG')
       },
-      "cirruslib": {
+      "cirrus.lib": {
           "handlers": ["standard"],
           "level": getenv('CIRRUS_LOG_LEVEL', 'DEBUG')
       }

cirruslib/transfer.py → src/cirrus/lib/transfer.py

@@ -8,7 +8,7 @@
 from copy import deepcopy
 from dateutil.parser import parse as dateparse
 from os import getenv, path as op
-from cirruslib.utils import get_path
+from cirrus.lib.utils import get_path
 
 from typing import Dict, Optional, List
 

src/cirruslib/__init__.py

@@ -0,0 +1,7 @@
+import warnings
+
+warnings.warn(
+    "Imports via 'cirruslib' will be removed in a future cirrus version. "
+    "Update all imports to instead use 'cirrus.lib'.",
+    DeprecationWarning,
+)

src/cirruslib/catalog.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

src/cirruslib/errors.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

src/cirruslib/logging.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

src/cirruslib/statedb.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

src/cirruslib/transfer.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

src/cirruslib/utils.py

@@ -0,0 +1,4 @@
+import importlib
+import sys
+module = __name__.split('.')[-1]
+sys.modules[__name__] = importlib.import_module(f'cirrus.lib.{module}')

tests/test_catalog.py

@@ -3,7 +3,7 @@
 import json
 import unittest
 
-from cirruslib import Catalog
+from cirrus.lib.catalog import Catalog
 
 testpath = os.path.dirname(__file__)
 

tests/test_statedb.py

@@ -10,7 +10,7 @@
 from copy import deepcopy
 from datetime import datetime
 from decimal import Decimal
-from cirruslib.statedb import StateDB, STATES
+from cirrus.lib.statedb import StateDB, STATES
 
 ## fixtures
 testpath = os.path.dirname(__file__)

tests/test_transfer.py

@@ -5,7 +5,7 @@
 import unittest
 
 from boto3utils import s3
-from cirruslib import transfer
+from cirrus.lib import transfer
 from shutil import rmtree
 
 testpath = f"{os.path.dirname(__file__)}/test_transfer"