refactor with docker and stub flask app

.gitignore

@@ -31,7 +31,7 @@ tests/data/process/gbif10k_2023-01-26_chunk-6001-7000_raw.csv
 tests/data/process/gbif10k_2023-01-26_chunk-7001-8000_raw.csv
 tests/data/process/gbif10k_2023-01-26_chunk-8001-9000_raw.csv
 tests/data/process/gbif10k_2023-01-26_chunk-9001-10000_raw.csv
-diversity_stats_2024_02_01.csv
-site_stats_2024_02_01.csv
-species_stats_2024_02_01.csv
+obsolete/data/diversity_stats_2024_02_01.csv
+obsolete/data/site_stats_2024_02_01.csv
+obsolete/data/species_stats_2024_02_01.csv
 obsolete/data/input/US-RIIS_MasterList_2021_annotated_2023-09.csv

Dockerfile

@@ -22,17 +22,19 @@ RUN mkdir -p /scratch-path/log \
  && mkdir -p /scratch-path/sessions \
  && chown -R bison.bison /scratch-path
 
+# Run everything as the user 'bison'
 WORKDIR /home/bison
 USER bison
 
 COPY --chown=bison:bison ./requirements.txt .
 
+# Set up a virtual environment
 RUN python3 -m venv venv \
  && venv/bin/pip install --upgrade pip \
  && venv/bin/pip install --no-cache-dir -r ./requirements.txt
 
-# This assumes that the bison repository is present on the host machine and \
-# docker is run from the top of directory (with bison subdir directly below).
+# This assumes that the bison repository is present on the host machine and
+# docker is run from the top of the repo directory (with bison subdir directly below).
 COPY --chown=bison:bison ./bison ./bison
 
 
@@ -47,6 +49,7 @@ ENV PYTHONDONTWRITEBYTECODE 1
 # Turns off buffering for easier container logging
 ENV PYTHONUNBUFFERED 1
 ENV FLASK_ENV=development
+# Logs in the window where the docker command is executed
 CMD venv/bin/python -m debugpy --listen 0.0.0.0:${DEBUG_PORT} -m ${FLASK_MANAGE} run --host=0.0.0.0
 
 
@@ -57,3 +60,27 @@ FROM base as flask
 COPY --chown=bison:bison ./flask_app ./flask_app
 ENV FLASK_ENV=production
 CMD venv/bin/python -m gunicorn -w 4 --bind 0.0.0.0:5000 ${FLASK_APP}
+
+# ........................................................
+# Frontend base image (for development)
+FROM node:16.10.0-buster as base-front-end
+
+LABEL maintainer="Specify Collections Consortium <github.com/specify>"
+
+USER node
+WORKDIR /home/node
+
+COPY --chown=node:node bison/frontend/js_src/package*.json ./
+RUN npm install
+
+RUN mkdir dist \
+ && chown node:node dist
+
+#COPY --chown=node:node sppy/frontend/js_src .
+
+
+# ........................................................
+# Frontend image (for production) from base-front-end
+FROM base-front-end as front-end
+
+RUN npm run build

bison/common/constants.py

@@ -32,7 +32,6 @@
 LOGFILE_BACKUP_COUNT = 5
 ERR_SEPARATOR = "------------"
 
-
 USER_DATA_TOKEN = "###SCRIPT_GOES_HERE###"
 
 

bison/common/util.py

@@ -11,7 +11,8 @@
 from logging.handlers import RotatingFileHandler
 import pandas
 import os
-from sys import maxsize
+import sys
+import traceback
 
 from bison.common.constants import (
     ENCODING, INSTANCE_TYPE, KEY_NAME, LOGFILE_MAX_BYTES, LOG_FORMAT, LOG_DATE_FORMAT,
@@ -759,7 +760,7 @@ def get_csv_dict_writer(
     if fmode not in ("w", "a"):
         raise Exception("File mode must be 'w' (write) or 'a' (append)")
     if ready_filename(csvfile, overwrite=overwrite):
-        csv.field_size_limit(maxsize)
+        csv.field_size_limit(sys.maxsize)
         try:
             f = open(csvfile, fmode, newline="", encoding=encoding)
         except Exception as e:
@@ -797,7 +798,7 @@ def get_csv_dict_reader(
         FileNotFoundError: on missing csvfile
         PermissionError: on improper permissions on csvfile
     """
-    csv.field_size_limit(maxsize)
+    csv.field_size_limit(sys.maxsize)
 
     if quote_none is True:
         quoting = csv.QUOTE_NONE
@@ -822,6 +823,100 @@ def get_csv_dict_reader(
     return rdr, f
 
 
+# ..........................
+def get_traceback():
+    """Get the traceback for this exception.
+
+    Returns:
+        trcbk: traceback of steps executed before an exception
+    """
+    exc_type, exc_val, this_traceback = sys.exc_info()
+    tb = traceback.format_exception(exc_type, exc_val, this_traceback)
+    tblines = []
+    cr = "\n"
+    for line in tb:
+        line = line.rstrip(cr)
+        parts = line.split(cr)
+        tblines.extend(parts)
+    trcbk = cr.join(tblines)
+    return trcbk
+
+
+# ...............................................
+def combine_errinfo(errinfo1, errinfo2):
+    """Combine 2 dictionaries with keys `error`, `warning` and `info`.
+
+    Args:
+        errinfo1: dictionary of errors
+        errinfo2: dictionary of errors
+
+    Returns:
+        dictionary of errors
+    """
+    errinfo = {}
+    for key in ("error", "warning", "info"):
+        try:
+            lst = errinfo1[key]
+        except KeyError:
+            lst = []
+        try:
+            lst2 = errinfo2[key]
+        except KeyError:
+            lst2 = []
+
+        if lst or lst2:
+            lst.extend(lst2)
+            errinfo[key] = lst
+    return errinfo
+
+
+# ...............................................
+def add_errinfo(errinfo, key, val_lst):
+    """Add to a dictionary with keys `error`, `warning` and `info`.
+
+    Args:
+        errinfo: dictionary of errors
+        key: error type, `error`, `warning` or `info`
+        val_lst: error message or list of errors
+
+    Returns:
+        updated dictionary of errors
+    """
+    if errinfo is None:
+        errinfo = {}
+    if key in ("error", "warning", "info"):
+        if isinstance(val_lst, str):
+            val_lst = [val_lst]
+        try:
+            errinfo[key].extend(val_lst)
+        except KeyError:
+            errinfo[key] = val_lst
+    return errinfo
+
+
+# ......................................................
+def prettify_object(print_obj):
+    """Format an object for output.
+
+    Args:
+        print_obj (obj): Object to pretty print in output
+
+    Returns:
+        formatted string representation of object
+
+    Note: this splits a string containing spaces in a list to multiple strings in the
+        list.
+    """
+    # Used only in local debugging
+    from io import StringIO
+    from pprint import pp
+
+    strm = StringIO()
+    pp(print_obj, stream=strm)
+    obj_str = strm.getvalue()
+    return obj_str
+
+
 # .............................................................................
 class BisonNameOp():
     """Class for constructing filenames following a pattern for different processes."""

bison/frontend/js_src/.eslintrc.js

@@ -0,0 +1,16 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: './tsconfig.eslint.json',
+  },
+  env: {
+    browser: true,
+    node: true,
+  },
+  extends: ['@maxxxxxdlp/eslint-config'],
+  rules: {
+    '@typescript-eslint/no-empty-interface': 'off',
+    '@next/next/no-img-element': 'off',
+  },
+};

bison/frontend/js_src/.stylelintrc.js

@@ -0,0 +1,6 @@
+'use strict';
+
+module.exports = {
+  extends: '@maxxxxxdlp/stylelint-config',
+  rules: {},
+};

bison/frontend/js_src/css/frontend.css

@@ -0,0 +1,4 @@
+section#occ,
+section#name {
+  overflow-x: auto;
+}

bison/frontend/js_src/css/header.css

@@ -0,0 +1,21 @@
+header {
+  display: flex;
+  align-items: baseline;
+  column-gap: var(--half-size);
+  justify-content: space-between;
+  flex-wrap: wrap;
+  font-weight: normal;
+  flex-direction: row-reverse;
+  font-size: calc(var(--size) * 1.2);
+  padding-bottom: var(--size);
+}
+
+@media only screen and (max-width: 767px) {
+  header {
+    flex-direction: column;
+  }
+}
+
+header img {
+  height: var(--double-size);
+}

bison/frontend/js_src/css/leaflet.css

@@ -0,0 +1,119 @@
+.leaflet-map-container {
+  height: 50vh;
+  width: 100%;
+}
+
+.leaflet-map {
+  height: 100%;
+}
+
+.leaflet-full-screen-toggle,
+.leaflet-print-map {
+  cursor: pointer !important;
+  background: var(--darken);
+}
+
+.leaflet-full-screen-toggle {
+  border: none;
+  padding: var(--half-size);
+  height: calc(var(--double-size) * 1.5);
+}
+
+.leaflet-full-screen-toggle img {
+  width: var(--double-size);
+  height: var(--double-size);
+}
+
+.full-screen {
+  height: 100vh;
+  overflow: hidden;
+}
+
+.leaflet-map-full-screen {
+  width: 100vw;
+  height: 100vh;
+  position: fixed !important;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  z-index: 20;
+}
+
+.full-screen .leaflet-map-container:not(.leaflet-map-full-screen),
+.full-screen table,
+.full-screen .alert {
+  display: none;
+}
+
+.leaflet-map-full-screen .leaflet-print-map {
+  font-size: var(--double-size);
+  padding: 0.1rem var(--half-size);
+}
+
+.leaflet-container:not(.leaflet-map-full-screen) .leaflet-print-map {
+  display: none;
+}
+
+.leaflet-control-layers-overlays > label > div {
+  display: flex;
+  column-gap: var(--half-size);
+  align-items: center;
+}
+
+.leaflet-control-layers-overlays > label > div > span {
+  display: contents;
+}
+
+.leaflet-control-layers-selector {
+  margin: 0;
+  padding: 0;
+  position: static;
+}
+
+.leaflet-legend-gradient {
+  --left-color: transparent;
+  --right-color: transparent;
+  background: linear-gradient(90deg, var(--left-color), var(--right-color));
+  display: flex;
+  justify-content: space-between;
+  padding: var(--quarter-size);
+  flex: 1;
+}
+
+.leaflet-legend-gradient:before {
+  content: '0';
+}
+
+.leaflet-legend-gradient:after {
+  content: '100+';
+}
+
+.leaflet-legend-point {
+  --color: transparent;
+  background: var(--color);
+  width: var(--size);
+  height: var(--size);
+  display: inline-block;
+  border-radius: 100vw;
+}
+
+.hue-rotate {
+  filter: hue-rotate(180deg) saturate(1.5) brightness(1.3);
+}
+
+.grayscale {
+  filter: grayscale(100%) brightness(150%);
+}
+
+.darkened {
+  filter: brightness(40%);
+}
+
+.saturated {
+  filter: saturate(150%);
+}
+
+.hyper-saturated {
+  filter: saturate(180%) brightness(1.3);
+}