use pytest

.gitignore

@@ -2,3 +2,4 @@
 
 .idea/
 venv/
+/.pytest_cache/

.pre-commit-config.yaml

@@ -0,0 +1,88 @@
+exclude: 'oldsrc'
+
+repos:
+#- repo: https://github.com/markdownlint/markdownlint
+#  rev: v0.11.0
+#  hooks:
+#    - id: markdownlint
+#      files: \.md$
+#      args:
+#        - --git-recurse
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+    -   id: check-added-large-files
+
+#-   repo: https://github.com/psf/black
+#    rev: 21.12b0
+#    hooks:
+#    -   id: black
+#        args: [--line-length=88]
+
+-   repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+    - id: flake8
+      args:
+        # these settings are needed to make flake8 compatible with black
+        - --max-line-length=88
+        - --select=C,E,F,W,B,B950
+        - --extend-ignore=E203,E501
+      additional_dependencies:
+        - flake8-bugbear
+
+-   repo: https://github.com/terrencepreilly/darglint
+    rev: v1.8.1
+    hooks:
+    - id: darglint
+      args: ['-v2', '-sgoogle']
+
+-   repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.1.1
+    hooks:
+    - id: pydocstyle
+      args: ['--convention=google']
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v2.29.1
+  hooks:
+    - id: pyupgrade
+
+- repo: https://github.com/asottile/setup-cfg-fmt
+  rev: v1.20.0
+  hooks:
+    - id: setup-cfg-fmt
+
+- repo: https://github.com/Lucas-C/pre-commit-hooks-safety
+  rev: v1.2.2
+  hooks:
+    - id: python-safety-dependencies-check
+
+# Checkitout
+- repo: https://github.com/Lucas-C/pre-commit-hooks-safety
+  rev: v1.2.2
+  hooks:
+    - id: python-safety-dependencies-check
+
+- repo: https://github.com/maxxxxxdlp/pre-commit
+  rev: v1.0.6
+  hooks:
+    - id: regex-blacklist
+      args:
+        - pre-commit-config/regex-blacklist.txt
+
+#-   repo: https://github.com/PyCQA/isort
+#    rev: 5.10.1
+#    hooks:
+#    - id: isort
+#      args: [--sp=pre-commit-config/.isort.cfg, --diff]
+#
+#-   repo: https://github.com/PyCQA/pylint
+#    rev: pylint-2.7.3
+#    hooks:
+#    - id: pylint
+#      args: [--rcfile=.github/linters/.python-lint]

.pydevproject

@@ -1,14 +1,14 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?eclipse-pydev version="1.0"?><pydev_project>
-    
+
     <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
-        
+
         <path>/${PROJECT_DIR_NAME}/src</path>
-        
+
     </pydev_pathproperty>
-    
+
     <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
-    
+
     <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
-    
+
 </pydev_project>

README.md

@@ -1,38 +1,12 @@
+[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
+
 # 2022 Data processing
 
 ## Data Inputs
 
 United States Register of Introduced and Invasive Species (US-RIIS)
 https://doi.org/10.5066/P95XL09Q
 
+## Test USGS input files
 
-# Obsolete data processing
-
-## Process BISON-provider and GBIF data for BISON
-
-.. _2018 Data Load: docs/notes/dataload_2018.rst
-.. _2019/2020 Data Load: docs/notes/dataload_2019-2020.rst
-.. _Data Load Run Dependencies: docs/envsetup/develop_run_env.rst
-.. _GBIF Data Processing: docs/notes/gbif_process.rst
-.. _BISON-Provider Data Processing: docs/notes/provider_dataset_process.rst
-.. _Common Data Processing: docs/notes/provider_dataset_process.rst
-.. _July 2019 Data load meeting: docs/notes/mtgnotes_2019_07.rst
-.. _Standup meeting decisions about KU processing: docs/notes/standup_mtg_decisions.rst
-
-## Data Load by year
-* `2019/2020 Data Load`_
-* `2018 Data Load`_
-
-## Latest processing specifics
-* `BISON-Provider Data Processing`_
-* `GBIF Data Processing`_
-* `Common Data Processing`_
-
-## Processing Dependencies
-
-see `Data Load Run Dependencies`_
-
-## Misc Notes
-
-* `July 2019 Data load meeting`_
-* `Standup meeting decisions about KU processing`_
+python3 test/test_RIIS.py

__init__.py

@@ -0,0 +1 @@
+"""Package for processing BISON data."""

data/.~lock.US-RIIS.xlsx#

@@ -0,0 +1 @@
+,astewart,badenov,27.12.2021 10:13,file:///home/astewart/.config/libreoffice/4;

data/US-RIIS_AuthorityReferences.csv

@@ -5949,4 +5949,4 @@ Zimmerman (1978)1of4,https://scholarspace.manoa.hawaii.edu/handle/10125/7338,Han
 Zimmerman (1978)2of4,https://scholarspace.manoa.hawaii.edu/handle/10125/7338,Handbook,University of Hawaii,"v. 9, Part I","Zimmerman, Elwood C.","Insects of Hawaii: A Manual of the Insects of the Hawaiian Islands, including an Enumeration of the Species and Notes on their Origin, Distribution, Hosts, Parasites, etc.",Insects of Hawaii,1978-01-01,University of Hawaii Press,"Honolulu, Hawaii",,,,
 Zimmerman (1978)3of4,https://scholarspace.manoa.hawaii.edu/handle/10125/7338,Handbook,University of Hawaii,"v. 9, Part II","Zimmerman, Elwood C.","Insects of Hawaii: A Manual of the Insects of the Hawaiian Islands, including an Enumeration of the Species and Notes on their Origin, Distribution, Hosts, Parasites, etc.",Insects of Hawaii,1978-01-01,University of Hawaii Press,"Honolulu, Hawaii",,,,
 Zimmerman (1978)4of4,https://scholarspace.manoa.hawaii.edu/handle/10125/7338,Handbook,University of Hawaii,"v. 9, Part II","Zimmerman, Elwood C.","Insects of Hawaii: A Manual of the Insects of the Hawaiian Islands, including an Enumeration of the Species and Notes on their Origin, Distribution, Hosts, Parasites, etc.",Insects of Hawaii,1978-01-01,University of Hawaii Press,"Honolulu, Hawaii",,,,
-Zimmerman et al. (1964),https://scholarspace.manoa.hawaii.edu/handle/10125/7340,Handbook,University of Hawaii,v. 11,"Zimmerman, Elwood C., D. Elmo Hardy, Marian A. Kohn, and Erwin Beyer","Insects of Hawaii: A Manual of the Insects of the Hawaiian Islands, including an Enumeration of the Species and Notes on their Origin, Distribution, Hosts, Parasites, etc.",Insects of Hawaii,1964-01-01,University of Hawaii Press,"Honolulu, Hawaii",,,,
+Zimmerman et al. (1964),https://scholarspace.manoa.hawaii.edu/handle/10125/7340,Handbook,University of Hawaii,v. 11,"Zimmerman, Elwood C., D. Elmo Hardy, Marian A. Kohn, and Erwin Beyer","Insects of Hawaii: A Manual of the Insects of the Hawaiian Islands, including an Enumeration of the Species and Notes on their Origin, Distribution, Hosts, Parasites, etc.",Insects of Hawaii,1964-01-01,University of Hawaii Press,"Honolulu, Hawaii",,,,

data/US-RIIS_DataDictionary.csv

@@ -55,4 +55,4 @@ Molluscabase,Molluscabase,text,Accepted Molluscabase,Used in taxonomicStatus.
 n/a,not applicable,text,n/a,Used throughout.
 NCBI,National Center for Biotechnology Information,text,Accepted NCBI,Used in taxonomicStatus.
 OToL,Open Tree of Life,text,Accepted OToL,Used in taxonomicStatus.
-uBIO,U.S. Marine Biological Laboratory at Woods Hole,text,Accepted uBIO,Used in taxonomicStatus.
+uBIO,U.S. Marine Biological Laboratory at Woods Hole,text,Accepted uBIO,Used in taxonomicStatus.

data/US-RIIS_MasterList.csv

@@ -15262,4 +15262,4 @@ L48,Zoysia matrella,(L.) Merr.,Manila grass,Species,Invasive,,,,,,Plantae,Trache
 L48,Zoysia tenuifolia,Thiele,Mascarene grass,Species,Introduced,,,,,Zoysia matrella,Plantae,Tracheophyta,Magnoliopsida,Poales,Poaceae,Accepted ITIS,42275,,USDA Natural Resources Conservation Service (NRCS) PLANTS Database (2021foi),https://plants.usda.gov/home/plantProfile?symbol=ZOTE,2017-11-21,2021-05-14,Added providedSciName,,USRIIS-L48-Plantae11397
 L48,Zygina flammigera,"(de Fourcroy, 1785)",Leafhopper,Species,Introduced,,,1950 (Aukema and McCullough (2009)),1950,Zygina flammingera,Animalia,Arthropoda,Insecta,Hemiptera,Cicadellidae,Accepted GBIF,,2029290,Mattson et al. (1994),https://www.nrs.fs.fed.us/pubs/gtr/gtr_nc169.pdf,2017-01-26,2021-10-29,Added scientificNameAuthorship,,USRIIS-L48-Animalia5054
 L48,Zygodon menziesii,(Schwagr.) Arn.,Zygodon moss,Species,Introduced,,,,,Codonoblepharon menziesii,Plantae,Bryophyta,Bryopsida,Orthotrichales,Orthotrichaceae,Accepted ITIS,1134833,,Essl et al. (2014),https://doi.org/10.1111/ecog.00905,2020-04-27,2020-11-29,,Extracted from Appendix 4,USRIIS-L48-Plantae12894
-L48,Zygophyllum fabago,L.,Syrian beancaper,Species,Invasive,,,,,,Plantae,Tracheophyta,Magnoliopsida,Zygophyllales,Zygophyllaceae,Accepted ITIS,29059,3189902,USDA Natural Resources Conservation Service (NRCS) PLANTS Database (2021foj),https://plants.usda.gov/home/plantProfile?symbol=ZYFA,2017-11-21,2017-11-21,,,USRIIS-L48-Plantae11398
+L48,Zygophyllum fabago,L.,Syrian beancaper,Species,Invasive,,,,,,Plantae,Tracheophyta,Magnoliopsida,Zygophyllales,Zygophyllaceae,Accepted ITIS,29059,3189902,USDA Natural Resources Conservation Service (NRCS) PLANTS Database (2021foj),https://plants.usda.gov/home/plantProfile?symbol=ZYFA,2017-11-21,2017-11-21,,,USRIIS-L48-Plantae11398

oldsrc/README.md

@@ -0,0 +1,41 @@
+# Obsolete data processing
+
+## Process BISON-provider and GBIF data for BISON
+
+## Data Load by year
+* [2018 Data Load](notes/dataload_2018.rst)
+* [2019/2020 Data Load](notes/dataload_2019-2020.rst)
+
+## Latest processing specifics
+* [BISON-Provider Data Processing](notes/provider_dataset_process.rst)
+* [GBIF Data Processing](notes/gbif_process.rst)
+* [Common Data Processing](notes/provider_dataset_process.rst)****
+
+## Processing Dependencies
+
+[Data Load Run Dependencies](notes/develop_run_env.rst)
+
+## Misc Notes
+
+* [July 2019 Data load meeting](notes/mtgnotes_2019_07.rst)
+* [Standup meeting decisions about KU processing](notes/standup_mtg_decisions.rst)
+
+## 2019 Data from Paul
+
+#### bison.tar.gz
+Occurrence data from BISON providers. CSV file created from Postgres table "solr" which 
+is used to populate SOLR index.
+
+#### provider.csv
+
+BISON file containing provider lookup values for providers contained in "bisonProvider" table.
+Data is used to simplify filter function in web app.
+
+BISON provider ~= GBIF publisher/organization
+
+#### resource.csv
+
+BISON file containing resource lookup values for resources contained in "bisonResource" table.
+Data is used to simplify BISON API filter function.
+
+BISON resource ~= GBIF dataset