Updating tooling in the build and qa pipeline (#460)

* Add ruff

* Address Ruff complains

* Remove isort, black and autoflake, as they are replaces by ruff

* Remove pylint as well

* Update pyright and mypy versions

* Cleanup project config

* Reorder imports to the ruff's preference

.pre-commit-config.yaml

@@ -7,61 +7,21 @@ repos:
       - id: check-toml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.3
+    hooks:
+      - id: ruff
+        args:
+        - --fix
+        - --exit-non-zero-on-fix
+      - id: ruff-format
   - repo: local
     hooks:
-      - id: autoflake
-        name: autoflake_tests
-        entry: poetry run autoflake
-        language: system
-        files: ^tests/.*
-      - id: autoflake
-        name: autoflake_docs
-        entry: poetry run autoflake
-        language: system
-        files: ^docs/source/code_examples/.*
-      - id: black
-        name: black
-        entry: poetry run black --config=pyproject.toml
-        types_or: [ python, pyi ]
-        language: system
-        files: ^encord/.*
-      - id: black
-        name: black_tests
-        entry: poetry run black --config=pyproject.toml
-        types_or: [ python, pyi ]
-        language: system
-        files: ^tests/.*
-      - id: black
-        name: black_docs
-        entry: poetry run black --config=docs/source/code_examples/pyproject.toml
-        types_or: [ python, pyi ]
-        language: system
-        files: ^docs/source/code_examples/.*
-      - id: isort
-        name: isort
-        entry: poetry run isort --sp=pyproject.toml
-        language: system
-        files: ^encord/.*py$
-      - id: isort
-        name: isort_tests
-        entry: poetry run isort --sp=pyproject.toml
-        language: system
-        files: ^tests/.*py$
-      - id: isort
-        name: isort_docs
-        entry: poetry run isort --sp=pyproject.toml
-        language: system
-        files: ^docs/source/code_examples/.*py$
       - id: pyright
         name: pyright
         entry: poetry run pyright
         language: system
         files: ^encord/.*py$
-      - id: pylint
-        name: pylint
-        entry: poetry run pylint
-        language: system
-        files: ^encord/.*
       - id: mypy
         name: mypy
         entry: poetry run mypy

encord/client.py

@@ -74,9 +74,6 @@
     AddPrivateDataResponse,
     DataRow,
     DataRows,
-)
-from encord.orm.dataset import Dataset as OrmDataset
-from encord.orm.dataset import (
     DatasetAccessSettings,
     DatasetData,
     DatasetDataLongPolling,
@@ -94,6 +91,7 @@
     SingleImage,
     Video,
 )
+from encord.orm.dataset import Dataset as OrmDataset
 from encord.orm.label_log import LabelLog, LabelLogParams
 from encord.orm.label_row import (
     AnnotationTaskStatus,
@@ -123,14 +121,12 @@
     CopyDatasetOptions,
     CopyLabelsOptions,
     CopyProjectPayload,
-)
-from encord.orm.project import Project as OrmProject
-from encord.orm.project import (
     ProjectCopy,
     ProjectCopyOptions,
     ProjectDataset,
     ProjectUsers,
 )
+from encord.orm.project import Project as OrmProject
 from encord.orm.workflow import (
     LabelWorkflowGraphNode,
     LabelWorkflowGraphNodePayload,

encord/configs.py

@@ -30,6 +30,7 @@
 
 import encord.exceptions
 from encord.constants.string_constants import ALL_RESOURCE_TYPES
+from encord.exceptions import ResourceNotFoundError
 from encord.http.constants import DEFAULT_REQUESTS_SETTINGS, RequestsSettings
 
 ENCORD_DOMAIN = "https://api.encord.com"
@@ -52,8 +53,6 @@
 
 logger = logging.getLogger(__name__)
 
-from encord.exceptions import ResourceNotFoundError
-
 
 class BaseConfig(ABC):
     def __init__(self, endpoint: str, requests_settings: RequestsSettings = DEFAULT_REQUESTS_SETTINGS):

encord/dataset.py

@@ -7,9 +7,9 @@
 from encord.constants.enums import DataType
 from encord.http.utils import CloudUploadSettings
 from encord.orm.cloud_integration import CloudIntegration
-from encord.orm.dataset import AddPrivateDataResponse, DataRow
-from encord.orm.dataset import Dataset as OrmDataset
 from encord.orm.dataset import (
+    AddPrivateDataResponse,
+    DataRow,
     DatasetAccessSettings,
     DatasetDataLongPolling,
     DatasetUser,
@@ -18,6 +18,7 @@
     ImageGroupOCR,
     StorageLocation,
 )
+from encord.orm.dataset import Dataset as OrmDataset
 
 
 class Dataset:

encord/http/error_utils.py

@@ -13,7 +13,31 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from encord.exceptions import *
+from encord.exceptions import (
+    AnswerDictionaryError,
+    AuthenticationError,
+    AuthorisationError,
+    CorruptedLabelError,
+    DetectionRangeInvalidError,
+    DuplicateSshKeyError,
+    FeatureDoesNotExistError,
+    FileSizeNotSupportedError,
+    FileTypeNotSupportedError,
+    GenericServerError,
+    InvalidArgumentsError,
+    InvalidDateFormatError,
+    MethodNotAllowedError,
+    ModelFeaturesInconsistentError,
+    ModelWeightsInconsistentError,
+    MultiLabelLimitError,
+    OperationNotAllowed,
+    ResourceExistsError,
+    ResourceNotFoundError,
+    SshKeyNotFound,
+    UnknownException,
+    UploadOperationNotSupportedError,
+    WrongProjectTypeError,
+)
 
 # Error messages
 AUTHENTICATION_ERROR = ["AUTHENTICATION_ERROR"]

encord/http/querier.py

@@ -18,7 +18,7 @@
 import random
 import uuid
 from contextlib import contextmanager
-from typing import Any, Generator, List, Tuple, Type, TypeVar
+from typing import Any, Generator, List, Optional, Tuple, Type, TypeVar
 
 import requests
 import requests.exceptions
@@ -27,7 +27,7 @@
 
 from encord._version import __version__ as encord_version
 from encord.configs import BaseConfig
-from encord.exceptions import *
+from encord.exceptions import RequestException, ResourceNotFoundError
 from encord.http.common import (
     HEADER_CLOUD_TRACE_CONTEXT,
     HEADER_USER_AGENT,

encord/http/v2/error_utils.py

@@ -1,4 +1,10 @@
-from encord.exceptions import *
+from encord.exceptions import (
+    AuthenticationError,
+    AuthorisationError,
+    MethodNotAllowedError,
+    ResourceNotFoundError,
+    UnknownException,
+)
 
 HTTP_UNAUTHORIZED = 401
 HTTP_FORBIDDEN = 403

encord/objects/project.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import List
+from typing import Any, Dict, List
 
 
 @dataclass
@@ -11,13 +11,13 @@ class ProjectDataset:
     description: str
 
     @staticmethod
-    def from_dict(d: dict) -> ProjectDataset:
+    def from_dict(d: Dict[str, Any]) -> ProjectDataset:
         return ProjectDataset(
             dataset_hash=d["dataset_hash"],
             title=d["title"],
             description=d["description"],
         )
 
     @classmethod
-    def from_list(cls, l: list) -> List[ProjectDataset]:
-        return [cls.from_dict(i) for i in l]
+    def from_list(cls, dataset_list: List[Dict[str, Any]]) -> List[ProjectDataset]:
+        return [cls.from_dict(i) for i in dataset_list]

encord/orm/base_dto/base_dto_pydantic_v2.py

@@ -1,8 +1,11 @@
 from typing import Any, Dict, Type, TypeVar
 
 # TODO: invent some dependency version dependent type checking to get rid of this ignore
-from pydantic import ConfigDict  # type: ignore[attr-defined]
-from pydantic import BaseModel, ValidationError
+from pydantic import (  # type: ignore[attr-defined]
+    BaseModel,
+    ConfigDict,  # type: ignore[attr-defined]
+    ValidationError,
+)
 
 from encord.common.utils import snake_to_camel
 from encord.exceptions import EncordException

encord/user_client.py

@@ -29,9 +29,9 @@
 )
 from encord.ontology import Ontology
 from encord.orm.cloud_integration import CloudIntegration
-from encord.orm.dataset import DEFAULT_DATASET_ACCESS_SETTINGS, CreateDatasetResponse
-from encord.orm.dataset import Dataset as OrmDataset
 from encord.orm.dataset import (
+    DEFAULT_DATASET_ACCESS_SETTINGS,
+    CreateDatasetResponse,
     DatasetAccessSettings,
     DatasetAPIKey,
     DatasetInfo,
@@ -41,21 +41,20 @@
     Images,
     StorageLocation,
 )
+from encord.orm.dataset import Dataset as OrmDataset
 from encord.orm.dataset_with_user_role import DatasetWithUserRole
 from encord.orm.ontology import Ontology as OrmOntology
 from encord.orm.project import (
     BenchmarkQaWorkflowSettings,
     CvatExportType,
     ManualReviewWorkflowSettings,
-)
-from encord.orm.project import Project as OrmProject
-from encord.orm.project import (
     ProjectImporter,
     ProjectImporterCvatInfo,
     ProjectWorkflowSettings,
     ProjectWorkflowType,
     ReviewMode,
 )
+from encord.orm.project import Project as OrmProject
 from encord.orm.project_api_key import ProjectAPIKey
 from encord.orm.project_with_user_role import ProjectWithUserRole
 from encord.project import Project