Add isort + sort files (#677)

Co-authored-by: Michael Franklin <[email protected]>

.pre-commit-config.yaml

@@ -19,6 +19,24 @@ repos:
       - id: markdownlint
         args: ["--config", ".markdownlint.json"]
 
+  - repo: https://github.com/populationgenomics/pre-commits
+    rev: "v0.1.3"
+    hooks:
+      - id: cpg-id-checker
+        args: ["--extra-pattern", 'TOB\d+']
+        exclude: >-
+          (?x)^(
+              test/test_generic_auditor\.py|
+              models/utils/sequencing_group_id_format\.py|
+              metamist/audit/README\.md
+          )$
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+
   - repo: https://github.com/ambv/black
     rev: 23.12.1
     hooks:

.pylintrc

@@ -23,7 +23,7 @@
 disable=f-string-without-interpolation,inherit-non-class,too-few-public-methods,
         fixme,logging-fstring-interpolation,import-error,missing-module-docstring,line-too-long,
         too-many-return-statements,no-name-in-module,R0801,consider-using-set-comprehension,
-        arguments-differ,unspecified-encoding,invalid-name,logging-not-lazy,I1101
+        arguments-differ,unspecified-encoding,invalid-name,logging-not-lazy,I1101,wrong-import-order
 
 # Overriding variable name patterns to allow short 1- or 2-letter variables
 attr-rgx=[a-z_][a-z0-9_]{0,30}$

api/graphql/filters.py

@@ -1,4 +1,4 @@
-from typing import TypeVar, Generic, Callable, Any
+from typing import Any, Callable, Generic, TypeVar
 
 import strawberry
 

api/routes/__init__.py

@@ -1,11 +1,11 @@
-from api.routes.sample import router as sample_router
-from api.routes.imports import router as import_router
 from api.routes.analysis import router as analysis_router
 from api.routes.assay import router as assay_router
-from api.routes.participant import router as participant_router
+from api.routes.billing import router as billing_router
+from api.routes.enum import router as enum_router
 from api.routes.family import router as family_router
+from api.routes.imports import router as import_router
+from api.routes.participant import router as participant_router
 from api.routes.project import router as project_router
-from api.routes.web import router as web_router
-from api.routes.enum import router as enum_router
+from api.routes.sample import router as sample_router
 from api.routes.sequencing_groups import router as sequencing_groups_router
-from api.routes.billing import router as billing_router
+from api.routes.web import router as web_router

api/routes/assay.py

@@ -3,20 +3,14 @@
 from fastapi import APIRouter
 
 from api.utils import get_project_readonly_connection
-from api.utils.db import (
-    Connection,
-    get_projectless_db_connection,
-)
+from api.utils.db import Connection, get_projectless_db_connection
+from db.python.layers.assay import AssayLayer
 from db.python.tables.assay import AssayFilter
 from db.python.tables.project import ProjectPermissionsTable
-from db.python.layers.assay import (
-    AssayLayer,
-)
 from db.python.utils import GenericFilter
 from models.models.assay import AssayUpsert
 from models.utils.sample_id_format import sample_id_transform_to_raw_list
 
-
 router = APIRouter(prefix='/assay', tags=['assay'])
 
 

api/routes/enum.py

@@ -1,7 +1,8 @@
-from typing import Type
 from inspect import isclass
+from typing import Type
 
 from fastapi import APIRouter
+
 from api.utils.db import get_projectless_db_connection
 from db.python import enum_tables
 from db.python.enum_tables.enums import EnumTable

api/routes/family.py

@@ -1,19 +1,19 @@
 # pylint: disable=invalid-name
-import io
-import csv
 import codecs
+import csv
+import io
 from datetime import date
 from typing import List, Optional
 
-from fastapi import APIRouter, UploadFile, File, Query
+from fastapi import APIRouter, File, Query, UploadFile
 from pydantic import BaseModel
 from starlette.responses import StreamingResponse
 
 from api.utils import get_projectless_db_connection
 from api.utils.db import (
+    Connection,
     get_project_readonly_connection,
     get_project_write_connection,
-    Connection,
 )
 from api.utils.export import ExportType
 from api.utils.extensions import guess_delimiter_by_upload_file_obj

api/routes/imports.py

@@ -1,15 +1,15 @@
-import csv
 import codecs
+import csv
 from typing import Optional
 
-from fastapi import APIRouter, UploadFile, File
+from fastapi import APIRouter, File, UploadFile
 
+from api.utils.db import Connection, get_project_write_connection
+from api.utils.extensions import guess_delimiter_by_upload_file_obj
 from db.python.layers.participant import (
-    ParticipantLayer,
     ExtraParticipantImporterHandler,
+    ParticipantLayer,
 )
-from api.utils.extensions import guess_delimiter_by_upload_file_obj
-from api.utils.db import get_project_write_connection, Connection
 
 router = APIRouter(prefix='/import', tags=['import'])
 

api/routes/participant.py

@@ -4,18 +4,16 @@
 
 from fastapi import APIRouter
 from fastapi.params import Query
-from starlette.responses import StreamingResponse, JSONResponse
+from starlette.responses import JSONResponse, StreamingResponse
 
 from api.utils import get_projectless_db_connection
 from api.utils.db import (
-    get_project_write_connection,
-    get_project_readonly_connection,
     Connection,
+    get_project_readonly_connection,
+    get_project_write_connection,
 )
 from api.utils.export import ExportType
-from db.python.layers.participant import (
-    ParticipantLayer,
-)
+from db.python.layers.participant import ParticipantLayer
 from models.models.participant import ParticipantUpsert
 from models.models.sample import sample_id_format
 

api/routes/sample.py

@@ -1,18 +1,17 @@
 from fastapi import APIRouter, Body
 
 from api.utils.db import (
-    get_project_write_connection,
-    get_project_readonly_connection,
     Connection,
+    get_project_readonly_connection,
+    get_project_write_connection,
     get_projectless_db_connection,
 )
 from db.python.layers.sample import SampleLayer
 from db.python.tables.project import ProjectPermissionsTable
 from models.models.sample import SampleUpsert
-from models.utils.sample_id_format import (
-    # Sample,
-    sample_id_transform_to_raw,
+from models.utils.sample_id_format import (  # Sample,
     sample_id_format,
+    sample_id_transform_to_raw,
     sample_id_transform_to_raw_list,
 )
 

api/routes/sequencing_groups.py

@@ -1,18 +1,18 @@
 from typing import Any
+
 from fastapi import APIRouter
 from pydantic import BaseModel
 
 from api.utils.db import (
-    get_project_readonly_connection,
     Connection,
-    get_projectless_db_connection,
+    get_project_readonly_connection,
     get_project_write_connection,
+    get_projectless_db_connection,
 )
 from db.python.layers.sequencing_group import SequencingGroupLayer
 from models.models.sequencing_group import SequencingGroupUpsertInternal
 from models.utils.sample_id_format import sample_id_format
-from models.utils.sequencing_group_id_format import (
-    # Sample,
+from models.utils.sequencing_group_id_format import (  # Sample,
     sequencing_group_id_format_list,
     sequencing_group_id_transform_to_raw,
 )

api/utils/__init__.py

@@ -1,13 +1,14 @@
 """Importing GCP libraries"""
 from collections import defaultdict
-from typing import TypeVar, Callable, Iterable
-from .openapi import get_openapi_schema_func
+from typing import Callable, Iterable, TypeVar
+
 from .db import (
     authenticate,
     get_project_readonly_connection,
     get_project_write_connection,
     get_projectless_db_connection,
 )
+from .openapi import get_openapi_schema_func
 
 T = TypeVar('T')
 X = TypeVar('X')

api/utils/dates.py

@@ -1,4 +1,4 @@
-from datetime import datetime, date, timedelta
+from datetime import date, datetime, timedelta
 
 INVOICE_DAY_DIFF = 3
 

api/utils/extensions.py

@@ -1,7 +1,6 @@
 import csv
 from typing import Optional
 
-
 EXTENSION_TO_DELIM_MAP = {
     '.csv': ',',
     '.tsv': '\t',

db/backup/backup.py

@@ -8,9 +8,7 @@
 from datetime import datetime
 from typing import Literal
 
-from google.cloud import storage
-from google.cloud import logging
-from google.cloud import secretmanager
+from google.cloud import logging, secretmanager, storage
 
 STORAGE_CLIENT = storage.Client()
 LOGGING_CLIENT = logging.Client()

db/backup/recovery_test.py

@@ -5,18 +5,18 @@
 
 """
 
-import unittest
-import subprocess
 import json
 import os
-from typing import Tuple, Optional
+import subprocess
+import unittest
 from collections import namedtuple
+from typing import Optional, Tuple
+
 import google.cloud.secretmanager
 import mysql.connector
 from parameterized import parameterized
 from restore import pull_latest_backup, restore
 
-
 BACKUP_BUCKET = 'cpg-sm-backups'
 LOCAL_BACKUP_FOLDER = 'latest_backup'
 

db/backup/restore.py

@@ -3,6 +3,7 @@
 
 import os
 import subprocess
+
 from google.cloud import storage
 
 BACKUP_BUCKET = 'cpg-sm-backups'

db/python/connect.py

@@ -70,9 +70,11 @@ async def audit_log_id(self):
 
         async with self._audit_log_lock:
             if not self._audit_log_id:
-                # pylint: disable=import-outside-toplevel
+
                 # make this import here, otherwise we'd have a circular import
-                from db.python.tables.audit_log import AuditLogTable
+                from db.python.tables.audit_log import (  # pylint: disable=import-outside-toplevel,R0401
+                    AuditLogTable,
+                )
 
                 at = AuditLogTable(self)
                 self._audit_log_id = await at.create_audit_log(
@@ -154,9 +156,9 @@ def get_connection_string(self):
         if self.port:
             _host += f':{self.port}'
 
-        options: dict[
-            str, str | int
-        ] = {}  # {'min_size': self.min_pool_size, 'max_size': self.max_pool_size}
+        options: dict[str, str | int] = (
+            {}
+        )  # {'min_size': self.min_pool_size, 'max_size': self.max_pool_size}
         _options = '&'.join(f'{k}={v}' for k, v in options.items())
 
         url = f'mysql://{u_p}@{_host}/{self.dbname}?{_options}'

db/python/gcp_connect.py

@@ -6,6 +6,7 @@
 
 import google.cloud.bigquery as bq
 from google.cloud import pubsub_v1
+
 from db.python.utils import InternalError
 
 logging.basicConfig(level=logging.DEBUG)

db/python/layers/assay.py

@@ -1,10 +1,10 @@
 # pylint: disable=too-many-arguments
 from typing import Any
 
-from db.python.utils import NoOpAenter
 from db.python.layers.base import BaseLayer, Connection
-from db.python.tables.assay import AssayTable, AssayFilter
+from db.python.tables.assay import AssayFilter, AssayTable
 from db.python.tables.sample import SampleTable
+from db.python.utils import NoOpAenter
 from models.models.assay import AssayInternal, AssayUpsertInternal
 
 

db/python/layers/search.py

@@ -1,13 +1,13 @@
 import asyncio
 from typing import List, Optional
 
-from db.python.utils import NotFoundError
 from db.python.layers.base import BaseLayer, Connection
 from db.python.tables.family import FamilyTable
 from db.python.tables.participant import ParticipantTable
 from db.python.tables.project import ProjectPermissionsTable
 from db.python.tables.sample import SampleTable
 from db.python.tables.sequencing_group import SequencingGroupTable
+from db.python.utils import NotFoundError
 from models.enums.search import SearchResponseType
 from models.models.sample import sample_id_format, sample_id_transform_to_raw
 from models.models.search import (

db/python/layers/seqr.py

@@ -12,6 +12,7 @@
 import slack_sdk
 import slack_sdk.errors
 from cloudpathlib import AnyPath
+
 from cpg_utils.cloud import get_google_identity_token
 
 from api.settings import (

etl/extract/main.py

@@ -7,10 +7,10 @@
 import flask
 import functions_framework
 import google.cloud.bigquery as bq
-from cpg_utils.cloud import email_from_id_token
-
 from google.cloud import pubsub_v1  # type: ignore
 
+from cpg_utils.cloud import email_from_id_token
+
 BIGQUERY_TABLE = os.getenv('BIGQUERY_TABLE')
 PUBSUB_TOPIC = os.getenv('PUBSUB_TOPIC')
 

etl/load/main.py

@@ -9,11 +9,8 @@
 import flask
 import functions_framework
 import google.cloud.bigquery as bq
-
-from google.cloud import pubsub_v1  # type: ignore
-
 import pkg_resources
-
+from google.cloud import pubsub_v1  # type: ignore
 
 BIGQUERY_TABLE = os.getenv('BIGQUERY_TABLE')
 BIGQUERY_LOG_TABLE = os.getenv('BIGQUERY_LOG_TABLE')