feat: target standalone server rather than Lambda

.env.example

@@ -0,0 +1,14 @@
+# .env for non-sensitive variables
+TRD_REDCAP_URL=
+TRD_MAILGUN_DOMAIN=
+TRD_MAILGUN_USERNAME=
+
+TRD_MAILTO_ADDRESS=
+TRD_LOG_DIR=
+TRD_LOG_LEVEL=
+
+# .env.local for secrets
+TRD_REDCAP_TOKEN=
+TRD_MAILGUN_SECRET=
+# Playground purposes only
+TRD_REDCAP_RESEARCHER_SECRET=

.github/workflows/test.yml

@@ -30,3 +30,8 @@ jobs:
               file: ./tests/coverage.xml
               flags: unittests
               fail_ci_if_error: false
+
+          - name: Check shell command
+            run: trd-cli --version
+          - name: Check export
+            run: trd-cli dump dump.txt

.gitignore

@@ -12,8 +12,8 @@ __pycache__/
 .venv
 
 .terraform/
-secrets.tfvars
-**/trd_cli.log
+.env.local
+**/trd_cli*.log
 
 tests/fixtures/rc_variables.txt
 

.pre-commit-config.yaml

@@ -18,9 +18,3 @@ repos:
       args: [ --fix ]
     # Run the formatter.
     - id: ruff-format
-
-- repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.96.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
-  hooks:
-    - id: terraform_fmt
-    - id: terraform_validate

README.md

@@ -7,37 +7,55 @@ This CLI performs automated data collection and curation for the TRD Database pr
 
 ## Deployment
 
-The tool is deployed to ECS on AWS by deploying it as a `docker` container.
-The `Dockerfile` is included in the repository.
+The tool is deployed to a standalone Oxford MSD IT server. 
+It runs regularly as a Cron job that runs the command `trd-cli run`.
 
-The AWS systems are set up using `terraform` and the configuration is included in the `.tf` files.
-The core setup is in the `aws.tf` file.
+The `trd-cli` command is installed as a system-wide command using the `setup.py` file in the repository.
 
-Variables are declared in the `variables.tf` file, and their actual values are exported in `secret.tfvars`.
-The contents of `secret.tfvars` are not included in the repository.
+## `trd-cli` command
 
-`secrets.tfvars` should be created in the root directory of the repository and should contain the following variables:
-```
-# secrets.tfvars
+The `trd-cli` command is the entry point for the tool.
+It has two subcommands: `run` and `dump`.
 
-redcap_secret        = "your_redcap_secret_value"
-true_colours_secret  = "your_true_colours_secret_value"
-mailgun_secret       = "your_mailgun_secret_value"
-```
+### `run`
+
+Run a single fetch-parse-compare-upload cycle for the True Colours and REDCap data.
+All configuration options may be specified as arguments to the command or as environment variables.
+For those that are marked as required, they must be specified in one of these ways.
+Command line arguments take precedence over environment variables.
+
+| Option          | Environment Variable       | Required | Description                                |
+|-----------------|----------------------------|----------|--------------------------------------------|
+| `--rc-url`      | `TRD_REDCAP_URL`           | Yes      | The URL of the REDCap API endpoint         |
+| `--rc-token`    | `TRD_REDCAP_TOKEN`         | Yes      | The API token for the REDCap project       |
+| `--tc-archive`  | `TRD_TRUE_COLOURS_ARCHIVE` | Yes      | File path to the True Colours data archive |
+| `--mailto`      | `TRD_MAILTO_ADDRESS`       | No       | The email address to send emails to        |
+| `--mg-secret`   | `TRD_MAILGUN_SECRET`       | No*      | The Mailgun API secret                     |
+| `--mg-domain`   | `TRD_MAILGUN_DOMAIN`       | No*      | The Mailgun domain                         |
+| `--mg-username` | `TRD_MAILGUN_USERNAME`     | No*      | The Mailgun username                       |
+| `--dry-run`     | _None_                     | No       | If set, the tool will not upload to REDCap |
+| `--log-dir`     | `TRD_LOG_DIR`              | No       | The directory to write log files to        |
+| `--log-level`   | `TRD_LOG_LEVEL`            | No       | The level of logging to use                |
+* Required if `mailto` is specified
+
+### `dump`
+
+Export the structure of the True Colours data to a file that can be used to create the REDCap project.
+This command has one option: `-o` or `--output`, which specifies the output file path.
+If the output file path is not specified, the output will be written to `stdout`.
 
 ## REDCap setup
 
 The project converts True Colours data to REDCap data.
 REDCap doesn't allow direct database access, however, so we need to create fake instruments in REDCap to store the data.
 
-
 ### The quick way
 
 The quick way to set up the REDCap project is still quite slow, but it's faster than the long way.
 
 #### Generate a list of variables
 
-Run `python trd_cli.py export_redcap_structure -o rc_variables.txt` to export the variables from the True Colours data.
+Run `trd-cli dump -o rc_variables.txt` to export the variables from the True Colours data.
 This will output a file called `rc_variables.txt` in the current directory.
 It will contain the names of the instruments and all the fields that will be exported for those instruments for 
 all the questionnaires the tool knows about.
@@ -206,7 +224,6 @@ pre-commit install
 
 The following hooks are used in this repository:
 - `ruff` Python code formatter and linter
-- `terraform` Terraform code formatter and linter
 - a few of the `pre-commit` default hooks
 
 ## Citations

requirements.txt

@@ -2,4 +2,5 @@
 click==8.1.7
 pycap==2.6.0
 requests==2.32.3
-typing_extensions==4.12.2
+typing_extensions==4.12.2
+setuptools==75.6.0

setup.py

@@ -0,0 +1,33 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="trd-cli",
+    version="0.1.0",
+    description="Treatment Resistant Depression Database Command Line Interface",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    author="Matt Jaquiery",
+    author_email="[email protected]",
+    url="https://github.com/OxfordRSE/trd-cli",
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        "click",
+        "requests",
+        "pandas",
+        "pytest",
+        "pytest-cov",
+        "codecov",
+    ],
+    entry_points={
+        "console_scripts": [
+            "trd-cli=trd_cli.main:cli",
+        ],
+    },
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.11",
+)

tests/playground.py

@@ -6,11 +6,11 @@
 
 
 class PlaygroundTestCase(unittest.TestCase):
-    redcap_secret = None
-    redcap_researcher_secret = None
+    REDCAP_TOKEN = None
+    REDCAP_RESEARCHER_SECRET = None
 
     def setUp(self):
-        with open("../secrets.tfvars", "r") as f:
+        with open("../.env.local", "r") as f:
             secrets = f.readlines()
             for line in secrets:
                 s = line.split("=")
@@ -26,21 +26,21 @@ def setUp(self):
                 self.exports[directory] = data
 
     def test_redcap_package(self):
-        project = Project("https://redcaptest.medsci.ox.ac.uk/api/", self.redcap_secret)
+        project = Project("https://redcaptest.medsci.ox.ac.uk/api/", self.REDCAP_TOKEN)
         records = project.export_records()
         self.assertGreater(len(records), 0)
         self.assertIn("nhsnumber", records[0])
 
     def test_redcap_package_researcher(self):
         project = Project(
-            "https://redcaptest.medsci.ox.ac.uk/api/", self.redcap_researcher_secret
+            "https://redcaptest.medsci.ox.ac.uk/api/", self.REDCAP_RESEARCHER_SECRET
         )
         records = project.export_records()
         self.assertGreater(len(records), 0)
         self.assertNotIn("nhsnumber", records[0])
 
     def test_redcap_package_import(self):
-        project = Project("https://redcaptest.medsci.ox.ac.uk/api/", self.redcap_secret)
+        project = Project("https://redcaptest.medsci.ox.ac.uk/api/", self.REDCAP_TOKEN)
         data = [
             {
                 "study_id": "102",