From 641d6ce382f4644969df455758252593a62d18ee Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 13:10:52 +0500 Subject: [PATCH 1/7] upd dependencies --- Makefile | 3 +++ dev-requirements.txt | 5 ++--- requirements.in | 3 +-- requirements.txt | 33 ++++++++++----------------------- 4 files changed, 16 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index a83d6fb..ada31f2 100644 --- a/Makefile +++ b/Makefile @@ -14,5 +14,8 @@ lint: flake8 *.py mypy +fmt: + isort . + dev: watchmedo auto-restart --patterns '*.py' python bot.py diff --git a/dev-requirements.txt b/dev-requirements.txt index f4d2ee0..9add27b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile dev-requirements.in # @@ -143,7 +143,6 @@ types-emoji==1.2.6 # via -r dev-requirements.in typing-extensions==3.10.0.0 # via - # boto3-stubs # flake8-pie # mypy watchdog[watchmedo]==2.1.6 diff --git a/requirements.in b/requirements.in index 735c7d6..a029df8 100644 --- a/requirements.in +++ b/requirements.in @@ -1,6 +1,5 @@ -python-telegram-bot +python-telegram-bot<20 python-dotenv -urlextract sentry-sdk peewee psycopg2-binary diff --git a/requirements.txt b/requirements.txt index bca09bf..0bf935d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,11 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # # pip-compile requirements.in # anyio==3.3.4 # via httpcore -appdirs==1.4.4 - # via urlextract apscheduler==3.6.3 # via python-telegram-bot boto3==1.20.5 @@ -20,26 +18,22 @@ cachetools==4.2.2 # via python-telegram-bot certifi==2021.5.30 # via + # httpcore # httpx # python-telegram-bot # sentry-sdk -charset-normalizer==2.0.7 - # via httpx emoji==1.6.1 # via -r requirements.in -filelock==3.0.12 - # via urlextract -h11==0.12.0 +h11==0.14.0 # via httpcore -httpcore==0.13.7 +httpcore==0.17.2 # via httpx -httpx==0.20.0 +httpx==0.24.1 # via -r requirements.in idna==3.2 # via # anyio - # rfc3986 - # urlextract + # httpx jmespath==0.10.0 # via # boto3 @@ -52,15 +46,12 @@ python-dateutil==2.8.2 # via botocore python-dotenv==0.17.1 # via -r requirements.in -python-telegram-bot==13.6 +python-telegram-bot==13.15 # via -r requirements.in -pytz==2021.1 +pytz==2023.3 # via # apscheduler # python-telegram-bot - # tzlocal -rfc3986[idna2008]==1.5.0 - # via httpx s3transfer==0.5.0 # via boto3 sentry-sdk==1.1.0 @@ -76,12 +67,8 @@ sniffio==1.2.0 # httpx tornado==6.1 # via python-telegram-bot -tzlocal==2.1 +tzlocal==5.0.1 # via apscheduler -uritools==3.0.2 - # via urlextract -urlextract==1.3.0 - # via -r requirements.in urllib3==1.26.5 # via # botocore From 444a38023c9dc4c9fd8804e74949ee422c771fa6 Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 13:11:09 +0500 Subject: [PATCH 2/7] add filter for test links --- filters.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/filters.py b/filters.py index aaa8f78..c129fce 100644 --- a/filters.py +++ b/filters.py @@ -2,9 +2,18 @@ from functools import reduce from telegram import Message from telegram.ext import BaseFilter, MessageFilter -from urlextract import URLExtract import text +from helpers import DB_ENABLED + +MIN_PREVIOUS_MESSAGES_COUNT = 3 + + +def is_fake_user(user_id: int) -> bool: + from models import LogEntry + + messages_count = LogEntry.select().where((LogEntry.user_id == user_id) & (LogEntry.action != 'delete')).count() + return messages_count < MIN_PREVIOUS_MESSAGES_COUNT class ChatMessageOnly(MessageFilter): @@ -34,14 +43,18 @@ def filter(self, message: Message) -> bool: class ContainsLink(MessageFilter): - def __init__(self) -> None: - self.extractor = URLExtract() def filter(self, message: Message) -> bool: if message.text is None: return False # type: ignore - return len(self.extractor.find_urls(message.text)) >= 1 + entities_types = set([entity.type for entity in message.entities]) + has_links = len(entities_types.intersection({'url', 'text_link'})) != 0 + + if DB_ENABLED() and has_links: + return is_fake_user(message.from_user.id) + + return has_links class ContainsThreeOrMoreEmojies(MessageFilter): From 80c886cd753e7e500bccbd60b8f83c0afe587cf7 Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 14:02:32 +0500 Subject: [PATCH 3/7] upd mypy --- mypy.ini | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mypy.ini b/mypy.ini index 55da233..8b52ac2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -python_version = 3.9 +python_version = 3.10 files = *.py warn_no_return = off warn_unused_configs = on @@ -17,6 +17,3 @@ ignore_missing_imports = on [mypy-playhouse.*] ignore_missing_imports = on - -[mypy-urlextract.*] -ignore_missing_imports = on From fb93f35a63e66f8dd4deaeb57fc2d7c99896e52d Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 15:25:50 +0500 Subject: [PATCH 4/7] default filter for previous messages --- .isort.cfg | 2 +- bot.py | 4 +--- filters.py | 30 +++++++++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index aa4633c..bc3cd63 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -line_length = 119 +line_length = 160 skip=migrations/*.py, node_modules, venv known_standard_library=typing multi_line_output=4 diff --git a/bot.py b/bot.py index 0f6b4f8..7f5b3c3 100644 --- a/bot.py +++ b/bot.py @@ -7,9 +7,7 @@ import rekognition import text -from filters import ( - ContainsLink, ContainsTelegramContact, ContainsThreeOrMoreEmojies, IsMedia, IsMessageOnBehalfOfChat, - with_default_filters) +from filters import ContainsLink, ContainsTelegramContact, ContainsThreeOrMoreEmojies, IsMedia, IsMessageOnBehalfOfChat, with_default_filters from helpers import DB_ENABLED, enable_logging, in_production, init_sentry diff --git a/filters.py b/filters.py index c129fce..c1e355e 100644 --- a/filters.py +++ b/filters.py @@ -6,14 +6,26 @@ import text from helpers import DB_ENABLED -MIN_PREVIOUS_MESSAGES_COUNT = 3 +class HasValidPreviousMessages(MessageFilter): + MIN_PREVIOUS_MESSAGES_COUNT = 3 -def is_fake_user(user_id: int) -> bool: - from models import LogEntry + def filter(self, message: Message) -> bool: + if not DB_ENABLED(): + return False + + return self.has_no_valid_previous_messages(user_id=message.from_user.id, chat_id=message.chat_id) + + @classmethod + def has_no_valid_previous_messages(cls, user_id: int, chat_id: int) -> bool: + from models import LogEntry - messages_count = LogEntry.select().where((LogEntry.user_id == user_id) & (LogEntry.action != 'delete')).count() - return messages_count < MIN_PREVIOUS_MESSAGES_COUNT + messages_count = LogEntry.select().where( + (LogEntry.user_id == user_id), + (LogEntry.chat_id == chat_id), + (LogEntry.action != 'delete'), + ).count() + return messages_count < cls.MIN_PREVIOUS_MESSAGES_COUNT class ChatMessageOnly(MessageFilter): @@ -25,6 +37,7 @@ def with_default_filters(*filters: BaseFilter) -> BaseFilter: """Apply default filters to the given filter classes""" default_filters = [ ChatMessageOnly(), + HasValidPreviousMessages(), ] return reduce(operator.and_, [*default_filters, *filters]) # МАМА Я УМЕЮ ФУНКЦИОНАЛЬНО ПРОГРАММИРОВАТЬ @@ -49,12 +62,7 @@ def filter(self, message: Message) -> bool: return False # type: ignore entities_types = set([entity.type for entity in message.entities]) - has_links = len(entities_types.intersection({'url', 'text_link'})) != 0 - - if DB_ENABLED() and has_links: - return is_fake_user(message.from_user.id) - - return has_links + return len(entities_types.intersection({'url', 'text_link'})) != 0 class ContainsThreeOrMoreEmojies(MessageFilter): From ebf14ae0b836317bf19487d07e256a072ab2af24 Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 15:36:35 +0500 Subject: [PATCH 5/7] use .python version --- .github/workflows/ci.yml | 7 +++++-- .python-version | 1 + Dockerfile | 3 ++- filters.py | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a7b260..0e9118b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,14 +9,15 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + id: setup-python with: - python-version: "3.10.9" + python-version-file: ".python-version" - uses: actions/cache@v3 with: path: | venv - key: ${{ runner.os }}-venv-v2-${{ hashFiles('**/*requirements.txt') }} + key: ${{ runner.os }}-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }} - name: Install dependencies if: steps.cache-primes.outputs.cache-hit != 'true' @@ -66,6 +67,8 @@ jobs: tags: | ghcr.io/${{ steps.image.outputs.lowercase }}:latest ghcr.io/${{ steps.image.outputs.lowercase }}:${{ github.sha }} + build-args: | + PYTHON_VERSION=${{ steps.setup-python.outputs.python-version }} deploy: needs: build-docker-image diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..d5cd4cc --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10.10 diff --git a/Dockerfile b/Dockerfile index ae05b2a..f48fd2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.10.9-slim-bullseye +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-bullseye RUN apt-get update && apt-get --no-install-recommends -y install wget && rm -Rf rm -rf /var/lib/apt/lists/* diff --git a/filters.py b/filters.py index c1e355e..c287c48 100644 --- a/filters.py +++ b/filters.py @@ -7,7 +7,7 @@ from helpers import DB_ENABLED -class HasValidPreviousMessages(MessageFilter): +class HasNoValidPreviousMessages(MessageFilter): MIN_PREVIOUS_MESSAGES_COUNT = 3 def filter(self, message: Message) -> bool: @@ -37,7 +37,7 @@ def with_default_filters(*filters: BaseFilter) -> BaseFilter: """Apply default filters to the given filter classes""" default_filters = [ ChatMessageOnly(), - HasValidPreviousMessages(), + HasNoValidPreviousMessages(), ] return reduce(operator.and_, [*default_filters, *filters]) # МАМА Я УМЕЮ ФУНКЦИОНАЛЬНО ПРОГРАММИРОВАТЬ From 6ca3bb74d3673d2abde96ec0732668d650ed7a63 Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 15:42:24 +0500 Subject: [PATCH 6/7] fix ci --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e9118b..400d5a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + id: setup-python + with: + python-version-file: '.python-version' + - name: Set up QEMU uses: docker/setup-qemu-action@v2 From b636840bf6cc58e8f08ebf14e01d02fd58bed337 Mon Sep 17 00:00:00 2001 From: Daniil Maltsev Date: Wed, 5 Jul 2023 18:00:17 +0500 Subject: [PATCH 7/7] turn on filter on users without prev messages without db --- filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filters.py b/filters.py index c287c48..47dbe87 100644 --- a/filters.py +++ b/filters.py @@ -12,7 +12,7 @@ class HasNoValidPreviousMessages(MessageFilter): def filter(self, message: Message) -> bool: if not DB_ENABLED(): - return False + return True return self.has_no_valid_previous_messages(user_id=message.from_user.id, chat_id=message.chat_id)