add lint workflow (#22)

* add lint workflow

* lint

* fix type check bug

* skip test data for spell check

* add codespell rule to pre commit config

* fix spelling

.github/workflows/lint.yaml

@@ -0,0 +1,14 @@
+name: lint
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -5,54 +5,25 @@ repos:
     hooks:
       - id: absolufy-imports
   - repo: https://github.com/python/black
-    rev: 22.10.0
+    rev: 23.10.1
     hooks:
     - id: black
-  # - repo: https://github.com/timothycrosley/isort
-  #   rev: 5.10.1
-  #   hooks:
-  #   - id: isort
-  #     additional_dependencies: [toml]
-  #     exclude: examples/*
-  # types
-  # - repo: https://github.com/pre-commit/mirrors-mypy
-  #   rev: 'v0.940'
-  #   hooks:
-  #   - id: mypy
-  #     language_version: python3
-  #     args: [--no-strict-optional, --ignore-missing-imports, --show-traceback, --install-types, --non-interactive]
-  #     exclude: docs/*
-  # code style
-  # - repo: https://github.com/pycqa/pylint
-  #   rev: pylint-2.7.4
-  #   hooks:
-  #   - id: pylint
-  #     exlude: notebooks/*
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.2
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.12.0
+    hooks:
+    - id: isort
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
     hooks:
     - id: flake8
-      exlude: notebooks/*
-  # notebooks
-  # - repo: https://github.com/s-weigand/flake8-nb
-  #   rev: v0.3.0
-  #   hooks:
-  #   - id: flake8-nb
-  #     files: \.ipynb$
-  # documentation
-  # - repo: https://github.com/econchick/interrogate
-  #   rev: 1.5.0
-  #   hooks:
-  #   - id: interrogate
-  #     exclude: ^(build|docs|merlin/io|tests|setup.py|versioneer.py)
-  #     args: [--config=pyproject.toml]
-  # - repo: https://github.com/codespell-project/codespell
-  #   rev: v2.1.0
-  #   hooks:
-  #   - id: codespell
-  # # security
-  # - repo: https://github.com/PyCQA/bandit
-  #   rev: 1.7.0
-  #   hooks:
-  #   - id: bandit
-  #     args: [--verbose, -ll, -x, tests,examples,bench]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+    - id: codespell
+      exclude: tests/testdata
+  # security
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.0
+    hooks:
+    - id: bandit
+      args: [--verbose, -ll, -x, tests,examples,bench]

ci/ignore_codespell_words.txt

@@ -0,0 +1 @@
+nin

crossfit/__init__.py

@@ -1,3 +1,5 @@
+# flake8: noqa
+
 from crossfit import backend, metric, op
 from crossfit.backend.dask.cluster import Distributed, Serial
 from crossfit.calculate.aggregate import Aggregator
@@ -6,9 +8,8 @@
 from crossfit.data.array.dispatch import crossarray
 from crossfit.data.dataframe.core import FrameBackend
 from crossfit.data.dataframe.dispatch import CrossFrame
-from crossfit.metric import *  # noqa
-from crossfit.op import *  # noqa
-
+from crossfit.metric import *
+from crossfit.op import *
 
 __all__ = [
     "Aggregator",
@@ -27,21 +28,23 @@
 
 
 try:
-    from crossfit.backend.torch import SentenceTransformerModel, TorchExactSearch, HFModel
+    from crossfit.backend.torch import HFModel, SentenceTransformerModel, TorchExactSearch
+    from crossfit.dataset.base import IRDataset, MultiDataset
+    from crossfit.dataset.load import load_dataset
     from crossfit.report.beir.embed import embed
     from crossfit.report.beir.report import beir_report
-    from crossfit.dataset.load import load_dataset
-    from crossfit.dataset.base import IRDataset, MultiDataset
-
-    __all__.extend([
-        "embed",
-        "beir_report",
-        "load_dataset",
-        "TorchExactSearch",
-        "SentenceTransformerModel",
-        "HFModel",    
-        "MultiDataset",
-        "IRDataset",
-    ])
+
+    __all__.extend(
+        [
+            "embed",
+            "beir_report",
+            "load_dataset",
+            "TorchExactSearch",
+            "SentenceTransformerModel",
+            "HFModel",
+            "MultiDataset",
+            "IRDataset",
+        ]
+    )
 except ImportError as e:
     pass

crossfit/backend/__init__.py

@@ -1,9 +1,10 @@
-from crossfit.backend.numpy.sparse import *
+# flake8: noqa
+
 from crossfit.backend.dask.dataframe import *
+from crossfit.backend.numpy.sparse import *
 from crossfit.backend.pandas.array import *
 from crossfit.backend.pandas.dataframe import *
 
-
 try:
     from crossfit.backend.cudf.array import *
     from crossfit.backend.cudf.dataframe import *

crossfit/backend/cudf/array.py

@@ -1,7 +1,7 @@
 import logging
 
 from crossfit.data.array import conversion
-from crossfit.data.array.dispatch import np_backend_dispatch, ArrayBackend
+from crossfit.data.array.dispatch import ArrayBackend, np_backend_dispatch
 
 
 @np_backend_dispatch.register_lazy("cudf")

crossfit/backend/cudf/dataframe.py

@@ -1,9 +1,8 @@
 from typing import Callable
 
-
+from crossfit.backend.pandas.dataframe import PandasDataFrame
 from crossfit.data.array.dispatch import crossarray
 from crossfit.data.dataframe.dispatch import CrossFrame
-from crossfit.backend.pandas.dataframe import PandasDataFrame
 
 
 class CudfDataFrame(PandasDataFrame):

crossfit/backend/cudf/series.py

@@ -1,5 +1,5 @@
-import cupy as cp
 import cudf
+import cupy as cp
 from cudf.core.column import as_column
 
 

crossfit/backend/cupy/array.py

@@ -1,7 +1,7 @@
 import logging
 
 from crossfit.data.array import conversion
-from crossfit.data.array.dispatch import np_backend_dispatch, ArrayBackend
+from crossfit.data.array.dispatch import ArrayBackend, np_backend_dispatch
 
 
 @np_backend_dispatch.register_lazy("cupy")

crossfit/backend/dask/aggregate.py

@@ -1,8 +1,8 @@
 from functools import partial
 
+import dask.dataframe as dd
 from dask.delayed import Delayed
 from dask.highlevelgraph import HighLevelGraph
-import dask.dataframe as dd
 
 from crossfit.calculate.aggregate import Aggregator
 from crossfit.data.dataframe.dispatch import CrossFrame

crossfit/backend/dask/cluster.py

@@ -1,17 +1,16 @@
-from typing import Callable, Optional, Any
-from contextvars import ContextVar
-import importlib
 import gc
+import importlib
 import warnings
+from contextvars import ContextVar
+from typing import Any, Callable, Optional
 
 import dask
-from dask.distributed import Client, get_client
-from dask.dataframe.optimize import optimize as dd_optimize
 import distributed
+from dask.dataframe.optimize import optimize as dd_optimize
+from dask.distributed import Client, get_client
 
 from crossfit.backend.gpu import HAS_GPU
 
-
 _crossfit_dask_client = ContextVar("_crossfit_dask_client", default="auto")
 
 
@@ -174,7 +173,7 @@ class only supports the automatic generation of
     The easiest way to use `Distributed` is within a
     conventional `with` statement::
 
-        from merlin.core.utils import Disributed
+        from merlin.core.utils import Disrtibuted
 
         workflow = nvt.Workflow(["col"] >> ops.Normalize())
         dataset = nvt.Dataset(...)

crossfit/backend/dask/dataframe.py

@@ -3,10 +3,10 @@
 from typing import Callable, List
 
 import dask.dataframe as dd
+
 from crossfit.data.dataframe.core import FrameBackend
 from crossfit.data.dataframe.dispatch import CrossFrame
 
-
 # @CrossFrame.register_lazy("dask")
 # def register_dask_backend():
 #     import dask.dataframe as dd

crossfit/backend/numpy/sparse.py

@@ -1,12 +1,12 @@
 import itertools
-from crossfit.data.array.masked import MaskedArray
 
+import numba
 import numpy as np
 import scipy.sparse as sp
-import numba
 
-from crossfit.data.sparse.dispatch import CrossSparse
+from crossfit.data.array.masked import MaskedArray
 from crossfit.data.sparse.core import SparseMatrixBackend
+from crossfit.data.sparse.dispatch import CrossSparse
 
 
 class NPSparseMatrixBackend(SparseMatrixBackend):

crossfit/backend/pandas/dataframe.py

@@ -36,7 +36,7 @@ def concat(cls, frames: List[FrameBackend], axis: int = 0):
         if len(frames) == 0:
             raise TypeError(f"Expected non-empty list, got {frames}")
         for frame in frames:
-            if type(frame) != cls:
+            if type(frame) is not cls:
                 raise TypeError(f"All frames should be type {cls}, got {type(frame)}")
 
         return cls(
@@ -103,8 +103,8 @@ def groupby_indices(self, by: list) -> dict:
 @CrossFrame.register_lazy("numpy")
 def register_numpy_backend():
     try:
-        import pandas as pd
         import numpy as np
+        import pandas as pd
 
         @CrossFrame.register(np.ndarray)
         def _numpy_to_pandas(data, name="data"):

crossfit/backend/torch/array.py

@@ -1,7 +1,7 @@
 import logging
 
 from crossfit.data.array import conversion
-from crossfit.data.array.dispatch import np_backend_dispatch, ArrayBackend
+from crossfit.data.array.dispatch import ArrayBackend, np_backend_dispatch
 
 try:
     import torch

crossfit/backend/torch/hf/model.py

@@ -1,15 +1,16 @@
-from functools import lru_cache
 import gc
 import os
-from crossfit.dataset.home import CF_HOME
-import joblib
+from functools import lru_cache
 
+import joblib
 import numpy as np
 import torch
+from sklearn.linear_model import LinearRegression
 from tqdm import tqdm
 from transformers import AutoConfig, AutoModel, AutoTokenizer
-from sklearn.linear_model import LinearRegression
+
 from crossfit.backend.torch.model import Model
+from crossfit.dataset.home import CF_HOME
 
 
 class HFModel(Model):
@@ -68,19 +69,13 @@ def fit_memory_estimate_curve(self, model=None):
                 torch.cuda.reset_peak_memory_stats()
 
                 batch = {
-                    "input_ids": torch.randint(1, 501, (batch_size, seq_len)).to(
-                        device=device
-                    ),
-                    "attention_mask": torch.ones((batch_size, seq_len)).to(
-                        device=device
-                    ),
+                    "input_ids": torch.randint(1, 501, (batch_size, seq_len)).to(device=device),
+                    "attention_mask": torch.ones((batch_size, seq_len)).to(device=device),
                 }
 
                 try:
-                    outputs = model(batch)
-                    memory_used = torch.cuda.max_memory_allocated() / (
-                        1024**2
-                    )  # Convert to MB
+                    _ = model(batch)
+                    memory_used = torch.cuda.max_memory_allocated() / (1024**2)  # Convert to MB
                     X.append([batch_size, seq_len, seq_len**2])
                     y.append(memory_used)