Improve set-up documentation (#534)

* Improve set-up documentation + aux changes Signed-off-by: Michael Franklin <[email protected]> * Fix default ped path for generate_data * Apply review feedback * Sneaky sneaky --------- Signed-off-by: Michael Franklin <[email protected]> Co-authored-by: Michael Franklin <[email protected]>
populationgenomics · Aug 28, 2023 · c5fa24e · c5fa24e
1 parent 38ac796
commit c5fa24e
Show file tree

Hide file tree

Showing 8 changed files with 244 additions and 238 deletions.
diff --git a/README.md b/README.md
diff --git a/api/settings.py b/api/settings.py
@@ -1,6 +1,7 @@
 # pylint: disable=global-statement
 import os
 from functools import lru_cache
+
 from cpg_utils.cloud import read_secret
 
 TRUTH_SET = ('1', 'y', 't', 'true')
@@ -10,6 +11,7 @@
 )
 _ALLOW_ALL_ACCESS: bool = os.getenv('SM_ALLOWALLACCESS', 'n').lower() in TRUTH_SET
 _DEFAULT_USER = os.getenv('SM_LOCALONLY_DEFAULTUSER')
+SM_ENVIRONMENT = os.getenv('SM_ENVIRONMENT', 'local').lower()
 SKIP_DATABASE_CONNECTION = bool(os.getenv('SM_SKIP_DATABASE_CONNECTION'))
 PROFILE_REQUESTS = os.getenv('SM_PROFILE_REQUESTS', 'false').lower() in TRUTH_SET
 IGNORE_GCP_CREDENTIALS_ERROR = os.getenv('SM_IGNORE_GCP_CREDENTIALS_ERROR') in TRUTH_SET

diff --git a/api/utils/openapi.py b/api/utils/openapi.py
@@ -1,13 +1,15 @@
 from os import getenv
+
 from fastapi.openapi.utils import get_openapi
 
-env = getenv('SM_ENVIRONMENT', 'local').lower()
+from api.settings import SM_ENVIRONMENT
+
 URLS = []
-if 'dev' in env:
+if 'dev' in SM_ENVIRONMENT:
     URLS.append('https://sample-metadata-dev.populationgenomics.org.au')
     URLS.append('https://sample-metadata-api-dev-mnrpw3mdza-ts.a.run.app')
 
-elif 'prod' in env:
+elif 'prod' in SM_ENVIRONMENT:
     URLS.append('https://sample-metadata.populationgenomics.org.au')
     URLS.append('https://sample-metadata-api-mnrpw3mdza-ts.a.run.app')
 else:

diff --git a/db/README.md b/db/README.md
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,11 +1,11 @@
 black>=22.3.0
+bump2version
 click==8.0.4
+coverage
 flake8
 flake8-bugbear
+nest-asyncio
 pre-commit
 pylint
-bump2version
 testcontainers[mariadb]
-nest-asyncio
-coverage
 types-PyMySQL
diff --git a/resources/debug-api.png b/resources/debug-api.png
diff --git a/test/README.md b/test/README.md
@@ -1,118 +1,19 @@
 # Running tests locally
 
-```bash
-```
-
-Clone the repo and install the env
-
-```bash
-git clone https://github.com/populationgenomics/sample-metadata
-cd sample-metadata
-virtualenv venv
-source venv/bin/activate
-pip install -r requirements.txt
-pip install -r requirements-dev.txt
-pip install --editable .
-```
-
-Start the DB server
-
-```bash
-docker stop mysql-p3307
-docker rm mysql-p3307
-docker run -p 3307:3306 --name mysql-p3307 -e MYSQL_ROOT_PASSWORD=root -d mariadb
-```
-
-Configure environment variables
-
-```bash
-# use credentials defined in db/python/connect.py:dev_config
-export SM_ENVIRONMENT=LOCAL
-# use specific mysql settings
-export SM_DEV_DB_PROJECT=sm_dev
-export SM_DEV_DB_USER=root
-export SM_DEV_DB_PORT=3307
-export SM_DEV_DB_HOST=127.0.0.1
-export SM_DEV_DB_PASSWORD=root
-```
-
-Create the DB
-
-```bash
-mysql --host=$SM_DEV_DB_HOST --port=$SM_DEV_DB_PORT -u $SM_DEV_DB_USER -p -e 'CREATE DATABASE '$SM_DEV_DB_PROJECT';'
-# mysql --host=$SM_DEV_DB_HOST --port=$SM_DEV_DB_PORT -u $SM_DEV_DB_USER -p -e 'show databases;'
-```
-
-Install tables
-
-```bash
-cd db
-liquibase update --url jdbc:mariadb://$SM_DEV_DB_HOST:$SM_DEV_DB_PORT/$SM_DEV_DB_PROJECT --username=$SM_DEV_DB_USER --password=$SM_DEV_DB_PASSWORD --classpath mariadb-java-client-2.7.3.jar --changelog-file=project.xml
-
-# mysql --host=$SM_DEV_DB_HOST --port=$SM_DEV_DB_PORT -u $SM_DEV_DB_USER -p -e 'use '$SM_DEV_DB_PROJECT'; show tables;'
-```
-
-Add project into the DB
-
-```bash
-INPUT_PROJECT=test_input_project
-OUTPUT_PROJECT=test_output_project
-[email protected]
-GCP_ID=vlad-dev
+Running tests requires docker (for mariadb), and all the regular dev dependencies.
 
-mysql --host=$SM_DEV_DB_HOST --port=$SM_DEV_DB_PORT -u $SM_DEV_DB_USER -p -e 'use '$SM_DEV_DB_PROJECT'; insert into project (id, name, author, dataset, gcp_id, read_secret_name, write_secret_name) values (1, "'$INPUT_PROJECT'", "'$USER'", "'$INPUT_PROJECT'", "'$GCP_ID'", "'$INPUT_PROJECT'-sample-metadata-main-read-members-cache", "'$INPUT_PROJECT'-sample-metadata-main-write-members-cache"), (2, "'$INPUT_PROJECT'", "'$USER'", "'$OUTPUT_PROJECT'", "'$GCP_ID'", "'$OUTPUT_PROJECT'-sample-metadata-main-read-members-cache", "'$OUTPUT_PROJECT'-sample-metadata-main-write-members-cache");'
+If you have these installed, you can run the tests on the terminal with:
 
-mysql --host=$SM_DEV_DB_HOST --port=$SM_DEV_DB_PORT -u $SM_DEV_DB_USER -p -e 'use '$SM_DEV_DB_PROJECT'; select * from project;'
+```shell
+python -m unittest discover -s test/
 ```
 
-Create secrets to test access to a project
+Otherwise, in VSCode:
 
-```bash
-# To read and NOT write input project:
-gcloud secrets create $INPUT_PROJECT-sample-metadata-main-read-members-cache --project $GCP_ID
-gcloud secrets create $INPUT_PROJECT-sample-metadata-main-write-members-cache --project $GCP_ID
+- Make sure your VSCode knows your python virtual environment version (`which python`)
+- Then from the "Testing" tab, you can "Configure Python Tests" with:
+    - `unittest`
+    - `test/` folder
+    - `test_*.py` format
 
-gcloud secrets versions add $INPUT_PROJECT-sample-metadata-main-read-members-cache --data-file=<(echo ,$USER,) --project $GCP_ID
-# Note empty user list for the write secret:
-gcloud secrets versions add $INPUT_PROJECT-sample-metadata-main-write-members-cache --data-file=<(echo ,) --project $GCP_ID
-
-# To read and write input project:
-gcloud secrets create $OUTPUT_PROJECT-sample-metadata-main-read-members-cache --project $GCP_ID
-gcloud secrets create $OUTPUT_PROJECT-sample-metadata-main-write-members-cache --project $GCP_ID
-
-gcloud secrets versions add $OUTPUT_PROJECT-sample-metadata-main-read-members-cache --data-file=<(echo ,$USER,) --project $GCP_ID
-gcloud secrets versions add $OUTPUT_PROJECT-sample-metadata-main-write-members-cache --data-file=<(echo ,$USER,) --project $GCP_ID
-```
-
-Generate and install API
-
-```bash
-python regenerate_api.py
-pip install -e .
-```
-
-Start the server to populate samples (can do in a separate window)
-
-```bash
-export SM_ALLOWALLACCESS=1
-python3 -m api.server
-```
-
-Populate samples
-
-```bash
-python test/test_add_samples_for_joint_calling.py
-```
-
-Stop the server and restart with SM_ALLOWALLACCESS unset, to test permissions
-
-```bash
-export SM_ALLOWALLACCESS=0
-python3 -m api.server
-```
-
-Run the test that simulates the joint-calling workflow
-
-```bash
-python test/test_joint_calling_workflow.py
-```
+This should display a full list of Python tests which you run all, or debug individual tests.
diff --git a/test/data/generate_data.py b/test/data/generate_data.py
@@ -4,27 +4,20 @@
 import asyncio
 import datetime
 import random
+from pathlib import Path
 from pprint import pprint
 
-from metamist.apis import (
-    ProjectApi,
-    ParticipantApi,
-    FamilyApi,
-    SampleApi,
-    AnalysisApi,
-)
+from metamist.apis import AnalysisApi, FamilyApi, ParticipantApi, ProjectApi, SampleApi
 from metamist.graphql import gql, query_async
 from metamist.model.analysis import Analysis
 from metamist.model.analysis_status import AnalysisStatus
-from metamist.models import (
-    SampleUpsert,
-    AssayUpsert,
-    SequencingGroupUpsert,
-)
+from metamist.models import AssayUpsert, SampleUpsert, SequencingGroupUpsert
 from metamist.parser.generic_parser import chunk
 
 EMOJIS = [':)', ':(', ':/', ':\'(']
 
+default_ped_location = str(Path(__file__).parent / 'greek-myth-forgeneration.ped')
+
 QUERY_SG_ID = gql(
     """
 query MyQuery($project: String!) {
@@ -51,7 +44,7 @@
 )
 
 
-async def main(ped_path='greek-myth-forgeneration.ped', project='greek-myth'):
+async def main(ped_path=default_ped_location, project='greek-myth'):
     """Doing the generation for you"""
 
     papi = ProjectApi()
@@ -234,7 +227,7 @@ def generate_random_number_within_distribution():
     parser.add_argument(
         '--ped-path',
         type=str,
-        default='greek-myth-forgeneration.ped',
+        default=default_ped_location,
         help='Path to the pedigree file',
     )
     parser.add_argument('--project', type=str, default='greek-myth')