Add text links filter

Makefile

@@ -14,5 +14,8 @@ lint:
 	flake8 *.py
 	mypy
 
+fmt:
+	isort .
+
 dev:
 	watchmedo auto-restart --patterns '*.py' python bot.py

dev-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
 #
 #    pip-compile dev-requirements.in
 #
@@ -143,7 +143,6 @@ types-emoji==1.2.6
     # via -r dev-requirements.in
 typing-extensions==3.10.0.0
     # via
-    #   boto3-stubs
     #   flake8-pie
     #   mypy
 watchdog[watchmedo]==2.1.6

filters.py

@@ -2,9 +2,18 @@
 from functools import reduce
 from telegram import Message
 from telegram.ext import BaseFilter, MessageFilter
-from urlextract import URLExtract
 
 import text
+from helpers import DB_ENABLED
+
+MIN_PREVIOUS_MESSAGES_COUNT = 3
+
+
+def is_fake_user(user_id: int) -> bool:
+    from models import LogEntry
+
+    messages_count = LogEntry.select().where((LogEntry.user_id == user_id) & (LogEntry.action != 'delete')).count()
+    return messages_count < MIN_PREVIOUS_MESSAGES_COUNT
 
 
 class ChatMessageOnly(MessageFilter):
@@ -34,14 +43,18 @@ def filter(self, message: Message) -> bool:
 
 
 class ContainsLink(MessageFilter):
-    def __init__(self) -> None:
-        self.extractor = URLExtract()
 
     def filter(self, message: Message) -> bool:
         if message.text is None:
             return False  # type: ignore
 
-        return len(self.extractor.find_urls(message.text)) >= 1
+        entities_types = set([entity.type for entity in message.entities])
+        has_links = len(entities_types.intersection({'url', 'text_link'})) != 0
+
+        if DB_ENABLED() and has_links:
+            return is_fake_user(message.from_user.id)
+
+        return has_links
 
 
 class ContainsThreeOrMoreEmojies(MessageFilter):

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.9
+python_version = 3.10
 files = *.py
 warn_no_return = off
 warn_unused_configs = on
@@ -17,6 +17,3 @@ ignore_missing_imports = on
 
 [mypy-playhouse.*]
 ignore_missing_imports = on
-
-[mypy-urlextract.*]
-ignore_missing_imports = on

requirements.in

@@ -1,6 +1,5 @@
-python-telegram-bot
+python-telegram-bot<20
 python-dotenv
-urlextract
 sentry-sdk
 peewee
 psycopg2-binary

requirements.txt

@@ -1,13 +1,11 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
 #
 #    pip-compile requirements.in
 #
 anyio==3.3.4
     # via httpcore
-appdirs==1.4.4
-    # via urlextract
 apscheduler==3.6.3
     # via python-telegram-bot
 boto3==1.20.5
@@ -20,26 +18,22 @@ cachetools==4.2.2
     # via python-telegram-bot
 certifi==2021.5.30
     # via
+    #   httpcore
     #   httpx
     #   python-telegram-bot
     #   sentry-sdk
-charset-normalizer==2.0.7
-    # via httpx
 emoji==1.6.1
     # via -r requirements.in
-filelock==3.0.12
-    # via urlextract
-h11==0.12.0
+h11==0.14.0
     # via httpcore
-httpcore==0.13.7
+httpcore==0.17.2
     # via httpx
-httpx==0.20.0
+httpx==0.24.1
     # via -r requirements.in
 idna==3.2
     # via
     #   anyio
-    #   rfc3986
-    #   urlextract
+    #   httpx
 jmespath==0.10.0
     # via
     #   boto3
@@ -52,15 +46,12 @@ python-dateutil==2.8.2
     # via botocore
 python-dotenv==0.17.1
     # via -r requirements.in
-python-telegram-bot==13.6
+python-telegram-bot==13.15
     # via -r requirements.in
-pytz==2021.1
+pytz==2023.3
     # via
     #   apscheduler
     #   python-telegram-bot
-    #   tzlocal
-rfc3986[idna2008]==1.5.0
-    # via httpx
 s3transfer==0.5.0
     # via boto3
 sentry-sdk==1.1.0
@@ -76,12 +67,8 @@ sniffio==1.2.0
     #   httpx
 tornado==6.1
     # via python-telegram-bot
-tzlocal==2.1
+tzlocal==5.0.1
     # via apscheduler
-uritools==3.0.2
-    # via urlextract
-urlextract==1.3.0
-    # via -r requirements.in
 urllib3==1.26.5
     # via
     #   botocore