feat: streamlined RC variables export

.gitignore

@@ -16,3 +16,5 @@ secrets.tfvars
 **/trd_cli.log
 
 tests/fixtures/rc_variables.txt
+
+tests/.redcap_structure.txt

README.md

@@ -35,9 +35,12 @@ REDCap doesn't allow direct database access, however, so we need to create fake
 
 The quick way to set up the REDCap project is still quite slow, but it's faster than the long way.
 
-1. Make sure you have all the questionnaires you need in your test dataset (`tests/fixtures/*.csv`).
-2. Run `tests/fixtures/tc_to_fixture.py`.
-3. Open the `tests/fixtures/rc_variables.txt` file created by the script.
+#### Generate a list of variables
+
+Run `python trd_cli.py export_redcap_structure -o rc_variables.txt` to export the variables from the True Colours data.
+This will output a file called `rc_variables.txt` in the current directory.
+It will contain the names of the instruments and all the fields that will be exported for those instruments for 
+all the questionnaires the tool knows about.
 
 The `rc_variables.txt` file will look like this:
 
@@ -49,7 +52,9 @@ field_3_name
 ...
 ```
 
-4. For each line with `######` at the start and end, create an instrument with the fields listed below it.
+#### Create the instruments in REDCap
+
+For each line with `######` at the start and end, create an instrument with the fields listed below it.
 
 You should use `instrument_name` as the name of the instrument in REDCap, although this is not strictly necessary.
 The field names must be copied exactly as they appear in the file.

tests/fixtures/tc_to_fixture.py

@@ -1,4 +1,3 @@
-from trd_cli.main_functions import compare_tc_to_rc
 
 
 def patient_to_fixture(patient_csv_data: dict) -> dict:
@@ -45,29 +44,3 @@ def tc_to_fixture(tc_data: dict) -> dict:
         json.dump(tc_to_fixture(parse_tc(".")), f, indent=4)
     with open("./tc_data_initial.json", "w+") as f:
         json.dump(parse_tc("./initial_tc_dump"), f, indent=4)
-
-    # Dump a list of the variables REDCap needs to have available to import data.
-    # This aids in setting up the REDCap project.
-    pp, rr = compare_tc_to_rc(parse_tc("."), redcap_id_data=list())
-    rc_variables_list = {}
-    for p in pp.values():
-        rc_variables_list["private"] = set(p["private"].keys())
-        rc_variables_list["info"] = set(p["info"].keys())
-        break
-    for r in rr:
-        if r["redcap_repeat_instrument"] not in rc_variables_list:
-            rc_variables_list[r["redcap_repeat_instrument"]] = set(r.keys())
-        else:
-            [rc_variables_list[r["redcap_repeat_instrument"]].add(k) for k in r.keys()]
-
-    redcap_vars = ["study_id", "redcap_repeat_instrument", "redcap_repeat_instance"]
-    for k, v in rc_variables_list.items():
-        rc_variables_list[k] = [x for x in v if x not in redcap_vars]
-    with open("./rc_variables.txt", "w+") as f:
-        f.flush()
-        for k in rc_variables_list.keys():
-            f.write(f"###### {k} ######\n")
-            vv = sorted(rc_variables_list[k])
-            for x in vv:
-                f.write(f"{x}\n")
-            f.write("\n")

tests/test_cli.py

@@ -7,11 +7,12 @@
 import subprocess
 import requests
 
-from trd_cli.conversions import QUESTIONNAIRES
-from trd_cli.main import cli
+from trd_cli.questionnaires import QUESTIONNAIRES
+from trd_cli.main import cli, export_redcap_structure
 from trd_cli.main_functions import compare_tc_to_rc
 
 cli: Command  # annotating to avoid linter warnings
+export_redcap_structure: Command
 
 
 class CliTest(TestCase):
@@ -198,6 +199,16 @@ def redcap_from_tc(tc_data: dict) -> List[dict]:
 
         self.assertEqual(result.exit_code, 0, result.output)
 
+    def test_export_redcap_structure(self):
+        """
+        Test exporting the REDCap structure to a file
+        """
+        runner = CliRunner()
+        result = runner.invoke(export_redcap_structure, "--output ./.redcap_structure.txt")
+
+        self.assertEqual(result.exit_code, 0, result.output)
+        self.assertTrue(os.path.exists("./.redcap_structure.txt"))
+
 
 if __name__ == "__main__":
     main()

tests/test_conversions.py

@@ -5,16 +5,14 @@
 
 from trd_cli.conversions import (
     extract_participant_info,
-    questionnaire_to_rc_record, get_code_by_name,
 )
+from trd_cli.questionnaires import questionnaire_to_rc_record, get_redcap_structure, get_code_by_name
 from trd_cli.parse_tc import parse_tc
 
 
 class ConversionsTest(TestCase):
-    def test_convert_scores(self):
-        responses = parse_tc("fixtures")
-        self.assertIn("questionnaireresponse.csv", responses)
-        expectations = {
+    def setUp(self):
+        self.expectations = {
             "phq9": {
                 "phq9_response_id": "1589930675",
                 "phq9_datetime": "2024-11-04T12:59:24.9477348+00:00",
@@ -79,6 +77,10 @@ def test_convert_scores(self):
                 "wsas_score_total_float": "21.0",
             },
         }
+
+    def test_convert_scores(self):
+        responses = parse_tc("fixtures")
+        self.assertIn("questionnaireresponse.csv", responses)
         done = []
         for r in responses["questionnaireresponse.csv"]:
             interop = r.get("interoperability")
@@ -88,17 +90,17 @@ def test_convert_scores(self):
                 if q_code is not None and q_code not in done:
                     with self.subTest(q_name=q_code):
                         done.append(q_code)
-                        if q_code not in expectations.keys():
+                        if q_code not in self.expectations.keys():
                             if q_code == "consent":
                                 continue
                             else:
                                 self.fail(
                                     f"Unhandled questionnaire response of type {q_code}."
                                 )
                         result = questionnaire_to_rc_record(r)
-                        self.assertEqual(expectations[q_code], result)
+                        self.assertEqual(self.expectations[q_code], result)
 
-        for q_code in expectations.keys():
+        for q_code in self.expectations.keys():
             if q_code not in done:
                 with self.subTest(q_name=q_code):
                     self.fail(f"Did not find a {q_code} row to test in data.")
@@ -142,6 +144,22 @@ def test_extract_info(self):
             public,
         )
 
+    def test_redcap_dump(self):
+        dump = get_redcap_structure()
+        self.assertIn("private", dump)
+        self.assertIsInstance(dump["private"], list)
+        for f in ["id", "nhsnumber", "birthdate", "contactemail", "mobilenumber", "firstname", "lastname", "preferredcontact"]:
+            self.assertIn(f, dump["private"])
+        self.assertIn("info", dump)
+        self.assertIsInstance(dump["info"], list)
+        for f in ["info_birthyear_int", "info_datetime", "info_deceased_datetime", "info_gender_int", "info_is_deceased_bool"]:
+            self.assertIn(f, dump["info"])
+        for k, v in self.expectations.items():
+            self.assertIn(k, dump)
+            self.assertIsInstance(dump[k], list)
+            for f in v.keys():
+                self.assertIn(f, dump[k])
+
 
 if __name__ == "__main__":
     main()

tests/test_main_functions.py

@@ -12,22 +12,36 @@ def test_parse_records(self):
         self.assertEqual(
             {
                 "one-oh-one": {
+                    'aq10': [],
+                    'asrs': [],
+                    'audit': [],
+                    'brss': [],
                     "consent": [],
                     "demo": [],
+                    'dudit': [],
                     "gad7": [("2222222", 1)],
                     "mania": [],
                     "phq9": [("1111111", 1), ("121212", 1)],
+                    'pvss': [],
                     "reqol10": [],
+                    'sapas': [],
                     "study_id": "101",
                     "wsas": [],
                 },
                 "one-oh-two": {
+                    'aq10': [],
+                    'asrs': [],
+                    'audit': [],
+                    'brss': [],
                     "consent": [],
                     "demo": [],
+                    'dudit': [],
                     "gad7": [("1231241", 1)],
                     "mania": [],
                     "phq9": [],
+                    'pvss': [],
                     "reqol10": [],
+                    'sapas': [],
                     "study_id": "102",
                     "wsas": [],
                 },

trd_cli/__init__.py

@@ -0,0 +1,12 @@
+__version__ = "0.1.0"
+__author__ = "Matt Jaquiery"
+__email__ = "[email protected]"
+__url__ = "https://github.com/OxfordRSE/trd-cli"
+__description__ = "A command line interface for converting True Colours data for REDCap import."
+
+import sys
+
+
+# Check for a minimum Python version
+if sys.version_info < (3, 9):
+    raise RuntimeError("This package requires Python 3.8 or higher.")