From 7587aac15153478bff7459d4190ae6b83f7a6c11 Mon Sep 17 00:00:00 2001 From: Bekzod Mirahmedov Date: Sun, 3 Mar 2024 17:59:26 +0500 Subject: [PATCH] add pytest-xdist and tests in CI --- .github/workflows/ci.yml | 35 ++++++++++++++++++++++++++++ .pre-commit-config.yaml | 14 ----------- Dockerfile | 2 +- Makefile | 3 +++ poetry.lock | 46 +++++++++++++++++++++++++++--------- pyproject.toml | 1 + src/config.py | 9 ++++++-- tests/conftest.py | 50 +++++++++++++++++++++++++++++----------- 8 files changed, 119 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0ce307..906b669 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.11" + cache: "poetry" - name: Install dependencies run: | poetry env use "3.11" @@ -19,3 +20,37 @@ jobs: pip install -r lint-requirements.txt - name: Run Ruff run: ruff check --output-format=github . + test: + name: Run Tests + needs: lint + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: ["5432:5432"] + redis: + image: redis:7.0.12-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + steps: + - uses: actions/checkout@v3 + - run: pipx install poetry + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + cache: "poetry" + - run: poetry install + - run: poetry run pytest -x -n auto --dist loadfile + env: + DB_HOST: "localhost" + REDIS_HOST: "localhost" + REDIS_PORT: "6379" \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60efe6a..48894ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,4 @@ repos: - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - - repo: https://github.com/psf/black - rev: 23.7.0 - hooks: - - id: black - - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - exclude: "src/tests/" - additional_dependencies: [Flake8-pyproject] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. rev: v0.3.0 diff --git a/Dockerfile b/Dockerfile index 87f29f5..2a69676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ WORKDIR /opt/chat COPY poetry.lock pyproject.toml ./ RUN pip install "poetry==$POETRY_VERSION" -RUN poetry export --with dev --output requirements.txt +RUN poetry export --with test,lint --output requirements.txt RUN pip install --no-deps -r requirements.txt COPY . . \ No newline at end of file diff --git a/Makefile b/Makefile index c1c33af..54a83ec 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,9 @@ down: test: docker exec -it chat-backend python -m pytest -svv $(target) +ftest: + docker exec -it chat-backend python -m pytest -x -n 2 --dist loadfile + test-integration: docker exec -it chat-backend python -m pytest -m "integration" -svv diff --git a/poetry.lock b/poetry.lock index c6cfa5b..7849b14 100644 --- a/poetry.lock +++ b/poetry.lock @@ -408,6 +408,20 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "execnet" +version = "2.0.2" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.7" +files = [ + {file = "execnet-2.0.2-py3-none-any.whl", hash = "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41"}, + {file = "execnet-2.0.2.tar.gz", hash = "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "fastapi" version = "0.103.2" @@ -729,16 +743,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1147,6 +1151,26 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-xdist" +version = "3.5.0" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-xdist-3.5.0.tar.gz", hash = "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a"}, + {file = "pytest_xdist-3.5.0-py3-none-any.whl", hash = "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"}, +] + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.2.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1625,4 +1649,4 @@ email = ["email-validator"] [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "3ab760f71082a0b0167d47a29a8de4a4d3e446093c8e7919b0fa65decddb8d2b" +content-hash = "a4972e4c5d792be9b26c79e7ce87a3e20fa06406a5f64b22f34fdcee67a47e58" diff --git a/pyproject.toml b/pyproject.toml index bfddf76..a6f4cbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ pytest = "7.4.3" pytest-asyncio = "0.23.5" pytest-env = "1.1.3" pytest-mock = "3.12.0" +pytest-xdist = "3.5.0" [tool.poetry.group.lint.dependencies] ruff = "0.3.0" diff --git a/src/config.py b/src/config.py index 76d1a82..94f5654 100644 --- a/src/config.py +++ b/src/config.py @@ -1,5 +1,6 @@ import logging import os +from random import randint import boto3 from pydantic_settings import BaseSettings, SettingsConfigDict @@ -58,7 +59,7 @@ class GlobalSettings(BaseSettings): class TestSettings(GlobalSettings): - DB_SCHEMA: str = "test" + DB_SCHEMA: str = f"test_{randint(1, 100)}" class DevelopmentSettings(GlobalSettings): @@ -125,6 +126,10 @@ def get_settings(): }, "loggers": { "": {"handlers": ["default"], "level": settings.LOG_LEVEL, "propagate": False}, - "uvicorn": {"handlers": ["default"], "level": logging.ERROR, "propagate": False}, + "uvicorn": { + "handlers": ["default"], + "level": logging.ERROR, + "propagate": False, + }, }, } diff --git a/tests/conftest.py b/tests/conftest.py index 383e43c..c319801 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,10 +21,14 @@ f"postgresql+asyncpg://" f"{settings.DB_USER}:{settings.DB_PASSWORD}@{settings.DB_HOST}:{settings.DB_PORT}/{settings.DB_NAME}" ) -engine_test = create_async_engine(DATABASE_URL_TEST, connect_args={"server_settings": {"jit": "off"}}) +engine_test = create_async_engine( + DATABASE_URL_TEST, connect_args={"server_settings": {"jit": "off"}} +) autocommit_engine = engine_test.execution_options(isolation_level="AUTOCOMMIT") -async_session_maker = sessionmaker(autocommit_engine, class_=AsyncSession, expire_on_commit=False) +async_session_maker = sessionmaker( + autocommit_engine, class_=AsyncSession, expire_on_commit=False +) metadata.bind = engine_test @@ -36,7 +40,7 @@ async def override_get_async_session() -> AsyncGenerator[AsyncSession, None]: app.dependency_overrides[get_async_session] = override_get_async_session -@pytest.fixture(scope="session") +@pytest.fixture() async def db_session(): async with autocommit_engine.begin() as conn: await conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {settings.DB_SCHEMA}")) @@ -140,15 +144,21 @@ def authenticated_doug_client(async_client: AsyncClient, doug_user: User): @pytest.fixture -async def bob_emily_chat(db_session: AsyncSession, bob_user: User, emily_user: User) -> Chat: +async def bob_emily_chat( + db_session: AsyncSession, bob_user: User, emily_user: User +) -> Chat: chat = Chat(chat_type=ChatType.DIRECT) chat.users.append(bob_user) chat.users.append(emily_user) db_session.add(chat) await db_session.flush() # make empty read statuses for both users last_read_message_id = 0 - initiator_read_status = ReadStatus(chat_id=chat.id, user_id=bob_user.id, last_read_message_id=0) - recipient_read_status = ReadStatus(chat_id=chat.id, user_id=emily_user.id, last_read_message_id=0) + initiator_read_status = ReadStatus( + chat_id=chat.id, user_id=bob_user.id, last_read_message_id=0 + ) + recipient_read_status = ReadStatus( + chat_id=chat.id, user_id=emily_user.id, last_read_message_id=0 + ) db_session.add_all([initiator_read_status, recipient_read_status]) await db_session.commit() @@ -156,15 +166,21 @@ async def bob_emily_chat(db_session: AsyncSession, bob_user: User, emily_user: U @pytest.fixture -async def bob_doug_chat(db_session: AsyncSession, doug_user: User, bob_user: User) -> Chat: +async def bob_doug_chat( + db_session: AsyncSession, doug_user: User, bob_user: User +) -> Chat: chat = Chat(chat_type=ChatType.DIRECT) chat.users.append(doug_user) chat.users.append(bob_user) db_session.add(chat) await db_session.flush() # make empty read statuses for both users last_read_message_id = 0 - initiator_read_status = ReadStatus(chat_id=chat.id, user_id=bob_user.id, last_read_message_id=0) - recipient_read_status = ReadStatus(chat_id=chat.id, user_id=doug_user.id, last_read_message_id=0) + initiator_read_status = ReadStatus( + chat_id=chat.id, user_id=bob_user.id, last_read_message_id=0 + ) + recipient_read_status = ReadStatus( + chat_id=chat.id, user_id=doug_user.id, last_read_message_id=0 + ) db_session.add_all([initiator_read_status, recipient_read_status]) await db_session.commit() @@ -199,12 +215,16 @@ async def bob_emily_chat_messages_history( @pytest.fixture async def bob_read_status( - db_session: AsyncSession, bob_user: User, bob_emily_chat_messages_history: list[Message] + db_session: AsyncSession, + bob_user: User, + bob_emily_chat_messages_history: list[Message], ) -> ReadStatus: # bob read 10 messages last_read_message = bob_emily_chat_messages_history[9] read_status = ReadStatus( - user_id=bob_user.id, chat_id=last_read_message.chat.id, last_read_message_id=last_read_message.id + user_id=bob_user.id, + chat_id=last_read_message.chat.id, + last_read_message_id=last_read_message.id, ) db_session.add(read_status) await db_session.commit() @@ -214,12 +234,16 @@ async def bob_read_status( @pytest.fixture async def emily_read_status( - db_session: AsyncSession, emily_user: User, bob_emily_chat_messages_history: list[Message] + db_session: AsyncSession, + emily_user: User, + bob_emily_chat_messages_history: list[Message], ) -> ReadStatus: # emily read 15 messages last_read_message = bob_emily_chat_messages_history[14] read_status = ReadStatus( - user_id=emily_user.id, chat_id=last_read_message.chat.id, last_read_message_id=last_read_message.id + user_id=emily_user.id, + chat_id=last_read_message.chat.id, + last_read_message_id=last_read_message.id, ) db_session.add(read_status) await db_session.commit()