feat: Updated Package Structure and Developer Tools

.codespell-whitelist

@@ -0,0 +1 @@
+kagisearch

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+  "image": "ghcr.io/swarm-io-internal/data-science-lite:latest",
+
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-toolsai.jupyter",
+        "ms-python.python"
+      ]
+    }
+  },
+  "postCreateCommand": "sh .devcontainer/postBuild.sh"
+}

.devcontainer/postBuild.sh

@@ -0,0 +1,4 @@
+# These commands will be run after the devcontainer is built.
+
+python3 -m pip install git+https://github.com/vioshyvo/mrpt/
+python3 -m pip install -e . # Install vectordb locally

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Raise pull requests for version updates
+    # to pip against the `main` branch
+    target-branch: "main"

.github/matchers/pylint.json

@@ -0,0 +1,32 @@
+{
+  "problemMatcher": [
+    {
+      "severity": "warning",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-warning"
+    },
+    {
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "code": 4,
+          "message": 5
+        }
+      ],
+      "owner": "pylint-error"
+    }
+  ]
+}

.github/workflows/build.yml

@@ -0,0 +1,90 @@
+# Any commit to master branch re-builds images, re-runs tests, and pushes SHA tags to DockerHub
+name: Build
+on:
+  push:
+    branches:
+      - main
+      - dev
+      - pre-commit
+    paths-ignore:
+      - 'LICENSE'
+      - 'README.md'
+  workflow_dispatch:
+
+env:
+  DOCKER_REGISTRY: ghcr.io
+  DOCKER_ORG: ${{ github.repository_owner }}
+  GITHUB_SHA: ${{ github.sha }}
+  GITHUB_REF: ${{ github.ref }}
+
+jobs:
+    build-images:
+      runs-on: ubuntu-latest
+      permissions:
+        contents: read
+        packages: write
+      strategy:
+        fail-fast: false
+        matrix:
+          IMAGE: [docker]
+      steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+
+      - name: Get date tag
+        id: get_date
+        run: |
+          DATE_TAG="$( date -u '+%Y.%m.%d' )"
+          echo "date_tag=$DATE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Get registry and org
+        id: registry_org
+        run: |
+          ORG=$(echo "${{ env.DOCKER_ORG }}" | tr '[:upper:]' '[:lower:]')
+          echo "image_base=${{ env.DOCKER_REGISTRY }}/${ORG}" >> $GITHUB_OUTPUT
+
+      # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+      - name: Free up disk space
+        run: |
+          df -h
+          docker image ls
+          sudo apt clean
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
+          df -h
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }}
+          tags: |
+            # set latest tag for default branch
+            type=raw,value=latest
+            type=raw,value=${{ steps.get_date.outputs.date_tag }}
+
+      - name: Log in to registry
+        uses: docker/login-action@v2
+        with:
+            registry: ${{ env.DOCKER_REGISTRY }}
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: ${{ matrix.IMAGE }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+
+      - name: Inspect Image
+        run: |
+            docker run ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }}:latest conda list
+            docker images ls
+
+      - name: Test Image
+        run: |
+            docker run -u 1000 -w /srv/test -v $PWD:/srv/test ${{ steps.registry_org.outputs.image_base }}/${{ matrix.IMAGE }}:latest

.github/workflows/ci.yml

@@ -0,0 +1,74 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: CI
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
+      - dev
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: 3
+  PROJECT_NAME: "vectordb"
+
+jobs:
+  build:
+    runs-on: ${{matrix.os}}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8","3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+
+    steps:
+      - name: Checkout vectordb
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - name: Record State
+        run: |
+          pwd
+          echo github.ref is: ${{ github.ref }}
+          echo GITHUB_SHA is: $GITHUB_SHA
+          echo github.event_name is: ${{ github.event_name }}
+          echo github workspace: ${{ github.workspace }}
+          pip --version
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest pytest-cov torch wheel
+
+      # We only want to install this on one run, because otherwise we'll have
+      # duplicate annotations.
+      - name: Install error reporter
+        if: ${{ matrix.python-version == '3.10' }}
+        run: |
+          python -m pip install pytest-github-actions-annotate-failures
+
+      - name: Install vectordb
+        run: |
+          pip install -e ".[dev]"
+          pip show ${{ env.PROJECT_NAME }}
+
+      - name: Test with pytest
+        run: |
+          pytest -vvv --cov=${{ env.PROJECT_NAME }} --cov-report=xml --cov-report=term tests/
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v4

.gitignore

@@ -160,4 +160,4 @@ cython_debug/
 #.idea/
 
 
-*~
+*~

.pre-commit-config.yaml

@@ -0,0 +1,85 @@
+exclude: |
+  (?x)^(
+    tests/utils/
+  )
+
+ci:
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+
+repos:
+  - repo: https://github.com/psf/black
+    rev: "24.2.0"
+    hooks:
+      - id: black-jupyter
+
+  - repo: https://github.com/asottile/blacken-docs
+    rev: "1.16.0"
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==23.7.0]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v4.5.0"
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v4.0.0-alpha.8"
+    hooks:
+      - id: prettier
+        types_or: [yaml, markdown, html, css, scss, javascript, json]
+        args: [--prose-wrap=always]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.3.2"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: "v1.9.0"
+  #   hooks:
+  #     - id: mypy
+  #       files: src
+  #       args: ["--ignore-missing-imports"]
+  #       additional_dependencies:
+  #         - pytest
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: "v2.2.6"
+    hooks:
+      - id: codespell
+        args: ["--write-changes", "--ignore-words", ".codespell-whitelist"]
+
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.7.1
+    hooks:
+      - id: nbstripout
+        args: [--extra-keys=metadata.kernelspec metadata.language_info.version]
+
+  - repo: local
+    hooks:
+      - id: disallow-caps
+        name: Disallow improper capitalization
+        language: pygrep
+        entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
+        exclude: .pre-commit-config.yaml