diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index 7e159db..21a1a19 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -117,13 +117,14 @@ jobs: ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/testing TELEGRAM_TEST_TOKEN: ${{ secrets.TELEGRAM_TEST_TOKEN }} TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }} + UPTRACE_DSN: ${{ secrets.UPTRACE_DSN }} with: host: ${{ secrets.MASTER_HOST }} username: ${{ secrets.MASTER_DO_USER }} passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }} key: ${{ secrets.MASTER_DO_SSH_KEY }} port: ${{ secrets.MASTER_PORT }} - envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID + envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN script: | export CONTAINER_ID=$(docker ps -aq --filter name=testing) export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:testing') @@ -135,8 +136,9 @@ jobs: touch ${{ env.ROOT_DIR }}/strings.sqlite touch ${{ env.ROOT_DIR }}/board_credentials.json docker run -dit --name sysblokbot-testing \ - --env APP_SOURCE="github CI" --restart unless-stopped \ + --env APP_SOURCE="testing" --restart unless-stopped \ --env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \ + --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \ --env TELEGRAM_TOKEN="${{ env.TELEGRAM_TEST_TOKEN }}" \ -v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \ -v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \ diff --git a/.github/workflows/publish_master.yml b/.github/workflows/publish_master.yml index a80e1ca..777a24c 100644 --- a/.github/workflows/publish_master.yml +++ b/.github/workflows/publish_master.yml @@ -120,13 +120,14 @@ jobs: ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/prod TELEGRAM_TOKEN: ${{ secrets.TELEGRAM_PROD_TOKEN }} TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }} + UPTRACE_DSN: ${{ secrets.UPTRACE_DSN }} with: host: ${{ secrets.MASTER_HOST }} username: ${{ secrets.MASTER_DO_USER }} passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }} key: ${{ secrets.MASTER_DO_SSH_KEY }} port: ${{ secrets.MASTER_PORT }} - envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID + envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN script: | export CONTAINER_ID=$(docker ps -aq --filter name=prod) export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:prod') @@ -138,9 +139,10 @@ jobs: touch ${{ env.ROOT_DIR }}/strings.sqlite touch ${{ env.ROOT_DIR }}/board_credentials.json docker run -dit --name sysblokbot-prod \ - --env APP_SOURCE="github CI" --restart unless-stopped \ + --env APP_SOURCE="prod" --restart unless-stopped \ --env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \ --env TELEGRAM_TOKEN="${{ env.TELEGRAM_TOKEN }}" \ + --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \ -v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \ -v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \ -v ${{ env.ROOT_DIR }}/sysblokbot.sqlite:/app/sysblokbot.sqlite \ diff --git a/.isort.cfg b/.isort.cfg index d8a9dc2..75bd081 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = apiclient,bs4,conftest,dateutil,deepdiff,facebook,fakes,freezegun,googleapiclient,matplotlib,numpy,oauth2client,pytest,pytest_report,requests,schedule,sentry_sdk,setuptools,sheetfu,sqlalchemy,telegram,telethon,utils,vk_api +known_third_party = apiclient,bs4,conftest,dateutil,deepdiff,facebook,fakes,freezegun,googleapiclient,matplotlib,numpy,oauth2client,pytest,pytest_report,requests,schedule,setuptools,sheetfu,sqlalchemy,telegram,telethon,utils,vk_api diff --git a/Dockerfile b/Dockerfile index 8256f5e..d9be45b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,10 @@ COPY . /app ARG COMMIT_HASH ARG COMMIT_HASH_SHORT +ARG UPTRACE_DSN ENV COMMIT_HASH=$COMMIT_HASH ENV COMMIT_HASH_SHORT=$COMMIT_HASH_SHORT +ENV UPTRACE_DSN=$UPTRACE_DSN ENV MUSL_LOCALE_DEPS cmake make musl-dev gcc gettext-dev libintl diff --git a/app.py b/app.py index 9dc94c2..204cda4 100644 --- a/app.py +++ b/app.py @@ -5,7 +5,6 @@ import logging import requests -import sentry_sdk from src import consts from src.bot import SysBlokBot @@ -13,6 +12,7 @@ from src.scheduler import JobScheduler from src.tg.sender import TelegramSender from src.utils.log_handler import ErrorBroadcastHandler +from src.utils.uptrace_logger import add_uptrace_logging locale.setlocale(locale.LC_TIME, "ru_RU.UTF-8") logging.basicConfig(format=consts.LOG_FORMAT, level=logging.INFO) @@ -34,10 +34,6 @@ def get_bot(): if not config: raise ValueError("Could not load config, can't go on") - sentry_dsn = config.get("sentry_dsn", None) - if sentry_dsn: - sentry_sdk.init(dsn=sentry_dsn, traces_sample_rate=1.0) - scheduler = JobScheduler() jobs_config_file_key = ConfigManager().get_jobs_config_file_key() @@ -66,6 +62,8 @@ def get_bot(): for handler in logging.getLogger().handlers: logging.getLogger().removeHandler(handler) logging.getLogger().addHandler(ErrorBroadcastHandler(tg_sender)) + if consts.UPTRACE_DSN: + add_uptrace_logging(consts.UPTRACE_DSN) # Scheduler must be run after clients initialized scheduler.run() @@ -82,7 +80,6 @@ def get_bot(): def report_critical_error(e: BaseException): - sentry_sdk.capture_exception(e) requests.post( url=f"https://api.telegram.org/bot{consts.TELEGRAM_TOKEN}/sendMessage", json={ diff --git a/requirements.txt b/requirements.txt index a20a2ee..bd2f1be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ cryptography==3.1.1 cycler==0.11.0 decorator==4.4.2 deepdiff==4.3.2 +Deprecated==1.2.14 distlib==0.3.6 distro==1.4.0 -e git+https://github.com/mobolic/facebook-sdk.git@ffd9980700be48964d6a6a61144edb1c3ea29cff#egg=facebook_sdk @@ -29,13 +30,14 @@ google-api-python-client==1.9.1 google-auth==1.16.0 google-auth-httplib2==0.0.3 google-auth-oauthlib==0.4.1 -googleapis-common-protos==1.51.0 +googleapis-common-protos==1.63.0 +grpcio==1.63.0 gspread==3.4.2 html5lib==1.0.1 httplib2==0.18.1 identify==2.5.11 idna==2.9 -importlib-metadata==1.6.0 +importlib-metadata==7.0.0 ipaddr==2.2.0 isort==4.3.21 kiwisolver==1.4.4 @@ -50,6 +52,14 @@ nodeenv==1.7.0 numpy==1.24.1 oauth2client==4.1.3 oauthlib==3.1.0 +opentelemetry-api==1.24.0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-proto==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 ordered-set==3.1.1 packaging==20.3 pathspec==0.10.3 @@ -59,7 +69,7 @@ platformdirs==2.6.2 pluggy==0.13.1 pre-commit==2.21.0 progress==1.5 -protobuf==3.12.2 +protobuf==4.25.3 py==1.8.1 pyaes==1.6.1 pyasn1==0.4.8 @@ -81,7 +91,6 @@ retrying==1.3.3 rope==0.17.0 rsa==4.0 -e git+https://github.com/dbader/schedule.git@3eac646a8d2658929587d7454bd2c85696df254e#egg=schedule -sentry-sdk==1.10.1 sheetfu==1.5.3 six==1.14.0 soupsieve==2.3.2.post1 diff --git a/src/bot.py b/src/bot.py index e62c8ec..dc4065c 100644 --- a/src/bot.py +++ b/src/bot.py @@ -26,7 +26,7 @@ from .tg import handlers, sender from .tg.handlers.utils import admin_only, direct_message_only, manager_only -logging.addLevelName(USAGE_LOG_LEVEL, "USAGE") +logging.addLevelName(USAGE_LOG_LEVEL, "NOTICE") def usage(self, message, *args, **kws): @@ -88,6 +88,12 @@ def init_handlers(self): self.manager_reply_handler("trello_board_state_job"), "получить сводку о состоянии доски", ) + self.add_manager_handler( + "get_board_state", + CommandCategories.SUMMARY, + self.manager_reply_handler("board_state_job"), + "получить сводку о состоянии доски (focalboard)", + ) self.add_manager_handler( "get_editorial_board_stats", CommandCategories.STATS, diff --git a/src/consts.py b/src/consts.py index 2d6e20d..81e3931 100644 --- a/src/consts.py +++ b/src/consts.py @@ -16,11 +16,13 @@ f'https://github.com/sysblok/sysblokbot/commit/{os.environ.get("COMMIT_HASH")}' ) COMMIT_HASH = os.environ.get("COMMIT_HASH_SHORT") +UPTRACE_DSN = os.environ.get("UPTRACE_DSN") class AppSource(Enum): DEFAULT = "manual" - GITHUB = "github CI" + GITHUB = "prod" + GITHUB_DEV = "testing" APP_SOURCE = os.environ.get("APP_SOURCE", AppSource.DEFAULT.value) @@ -86,6 +88,19 @@ class TrelloCardColor(Enum): UNKNOWN = "unknown" +class BoardCardColor(Enum): + BLACK = "propColorGray" + BROWN = "propColorBrown" + ORANGE = "propColorOrange" + YELLOW = "propColorYellow" + GREEN = "propColorGreen" + BLUE = "propColorBlue" + PURPLE = "propColorPurple" + PINK = "propColorPink" + RED = "propColorRed" + UNKNOWN = "unknown" + + class TrelloListAlias(Enum): TOPIC_SUGGESTION = "trello_list_name__topic_suggestion" TOPIC_READY = "trello_list_name__topic_ready" @@ -97,6 +112,7 @@ class TrelloListAlias(Enum): TO_CHIEF_EDITOR = "trello_list_name__to_chief_editor" PROOFREADING = "trello_list_name__proofreading" DONE = "trello_list_name__typesetting" + PUBLISHED = "trello_list_name__published" BACK_BURNER = "trello_list_name__back_burner" diff --git a/src/db/db_objects.py b/src/db/db_objects.py index a4723ae..35d159f 100644 --- a/src/db/db_objects.py +++ b/src/db/db_objects.py @@ -110,6 +110,7 @@ class TeamMember(Base): manager = Column(String) telegram = Column(String) trello = Column(String) + focalboard = Column(String) roles = Column(String) def __repr__(self): @@ -125,6 +126,7 @@ def from_dict(cls, data): member.manager = _get_str_data_item(data, "manager") member.telegram = _get_str_data_item(data, "telegram") member.trello = _get_str_data_item(data, "trello") + member.focalboard = _get_str_data_item(data, "focalboard") return member def to_dict(self): @@ -136,6 +138,7 @@ def to_dict(self): "manager": self.manager, "telegram": self.telegram, "trello": self.trello, + "focalboard": self.focalboard, "roles": self.roles, } @@ -149,6 +152,7 @@ def from_sheetfu_item(cls, item): member.manager = item.get_field_value(load("sheets__team__manager")) member.telegram = item.get_field_value(load("sheets__team__telegram")) member.trello = item.get_field_value(load("sheets__team__trello")) + member.focalboard = item.get_field_value(load("sheets__focalboard")) return member diff --git a/src/focalboard/focalboard_client.py b/src/focalboard/focalboard_client.py index 16904d3..85d0f53 100644 --- a/src/focalboard/focalboard_client.py +++ b/src/focalboard/focalboard_client.py @@ -1,3 +1,4 @@ +import ast import json import logging from typing import List @@ -40,7 +41,7 @@ def get_lists(self, board_id=None, sorted=False): for prop in [board for board in data if board["id"] == board_id][0][ "cardProperties" ] - if prop["name"] == "List" + if prop["name"] == "Колонка" ][0] lists_data = list_data["options"] lists = [ @@ -50,8 +51,12 @@ def get_lists(self, board_id=None, sorted=False): if sorted: # we need to get sorting order from the view, which is currently not efficient try: - _, data = self._make_request(f"api/v2/boards/{board_id}/blocks?all=true") - view = [card_dict for card_dict in data if card_dict["type"] == "view"][0] + _, data = self._make_request( + f"api/v2/boards/{board_id}/blocks?all=true" + ) + view = [card_dict for card_dict in data if card_dict["type"] == "view"][ + 0 + ] order = view["fields"]["visibleOptionIds"] sorted_lists = [] for list_id in order: @@ -59,7 +64,7 @@ def get_lists(self, board_id=None, sorted=False): sorted_lists.append(this_list) lists = sorted_lists except Exception as e: - logger.error(f"can't sort focalboard lists", exc_info=e) + logger.error("can't sort focalboard lists", exc_info=e) logger.debug(f"get_lists: {lists}") return lists @@ -70,7 +75,7 @@ def get_list(self, board_id, list_id): for prop in [board for board in data if board["id"] == board_id][0][ "cardProperties" ] - if prop["name"] == "List" + if prop["name"] == "Колонка" ][0]["options"] lst = [ objects.TrelloList.from_focalboard_dict(trello_list, board_id) @@ -80,6 +85,25 @@ def get_list(self, board_id, list_id): logger.debug(f"get_list: {lst}") return lst + def _get_labels(self, board_id=None): + if board_id is None: + # default board + board_id = self.board_id + _, data = self._make_request("api/v2/teams/0/boards") + label_data = [ + prop + for prop in [board for board in data if board["id"] == board_id][0][ + "cardProperties" + ] + if prop["name"] == "Рубрика" + ][0] + labels_data = label_data["options"] + labels = [ + objects.TrelloCardLabel.from_focalboard_dict(label) + for label in labels_data + ] + return labels + def _get_list_property(self, board_id): _, data = self._make_request("api/v2/teams/0/boards") return [ @@ -87,9 +111,47 @@ def _get_list_property(self, board_id): for prop in [board for board in data if board["id"] == board_id][0][ "cardProperties" ] - if prop["name"] == "List" + if prop["name"] == "Колонка" + ][0]["id"] + + def _get_label_property(self, board_id=None): + if board_id is None: + # default board + board_id = self.board_id + _, data = self._make_request("api/v2/teams/0/boards") + return [ + prop + for prop in [board for board in data if board["id"] == board_id][0][ + "cardProperties" + ] + if prop["name"] == "Рубрика" + ][0]["id"] + + def _get_due_property(self, board_id=None): + if board_id is None: + # default board + board_id = self.board_id + _, data = self._make_request("api/v2/teams/0/boards") + return [ + prop + for prop in [board for board in data if board["id"] == board_id][0][ + "cardProperties" + ] + if prop["name"] == "Дедлайн" ][0]["id"] + def get_card_due(self, card_id: str): + _, data = self._make_request(f"api/v2/cards/{card_id}") + due_id = self._get_due_property() + due_value = None + + fields = data["properties"] + for type_id, value in fields.items(): + if type_id == due_id: + due_value = value + due_value_dict = ast.literal_eval(due_value) + return due_value_dict.get("from", []) + def _get_member_property(self, board_id): _, data = self._make_request("api/v2/teams/0/boards") return [ @@ -97,7 +159,7 @@ def _get_member_property(self, board_id): for prop in [board for board in data if board["id"] == board_id][0][ "cardProperties" ] - if prop["name"] == "Assignee" + if prop["name"] == "Ответственный" ][0]["id"] def get_list_id_from_aliases(self, list_aliases): @@ -143,20 +205,59 @@ def get_board_custom_field_types(self): logger.debug(f"get_board_custom_field_types: {custom_field_types}") return custom_field_types - def get_board_custom_fields_dict(self, card_id): - custom_fields = self.get_board_custom_field_types() - custom_fields_dict = {} - for alias, type_id in self.custom_fields_config.items(): - suitable_fields = [fld for fld in custom_fields if fld.id == type_id] - if len(suitable_fields) > 0: - custom_fields_dict[alias] = suitable_fields[0] - return custom_fields_dict + def get_username(self, user_id: str): + _, data = self._make_request(f"api/v2/users/{user_id}") + username = data["username"] + return username def get_custom_fields(self, card_id: str) -> objects.CardCustomFields: card_fields = objects.CardCustomFields(card_id) - card_fields_dict = self.get_board_custom_fields_dict(card_id) - card_fields._data = card_fields_dict - logger.info(card_fields) + board_labels = self._get_labels() + card_fields_dict = {} + card_labels_ids = [] + card_labels = [] + _, data = self._make_request(f"api/v2/cards/{card_id}") + fields = data["properties"] + for alias, type_id in self.custom_fields_config.items(): + if type_id in fields: + changed_alias = alias.name.split(".")[-1].lower() + card_fields_dict[changed_alias] = fields[type_id] + + board_label_id = self._get_label_property() + + for type_id, value in fields.items(): + if type_id == board_label_id: + card_labels_ids = value + + for card_label_id in card_labels_ids: + for board_label in board_labels: + if card_label_id == board_label.id: + card_labels.append(board_label) + + card_fields.authors = [ + author.strip() + for author in card_fields_dict.get("author", []) + ] + card_fields.editors = [ + editor.strip() + for editor in card_fields_dict.get("editor", []) + ] + card_fields.illustrators = [ + illustrator.strip() + for illustrator in card_fields_dict.get("illustrator", []) + ] + card_fields.cover = ( + card_fields_dict["cover"] if "cover" in card_fields_dict else None + ) + card_fields.google_doc = ( + card_fields_dict["google_doc"] if "google_doc" in card_fields_dict else None + ) + card_fields.title = ( + card_fields_dict["title"] if "title" in card_fields_dict else None + ) + + card_fields._data = card_labels + return card_fields def get_members(self, board_id) -> List[objects.TrelloMember]: _, data = self._make_request(f"api/v2/boards/{board_id}/members") @@ -167,7 +268,7 @@ def get_members(self, board_id) -> List[objects.TrelloMember]: logger.debug(f"get_members: {members}") return members - def get_cards(self, list_ids, board_id=None): + def get_cards(self, list_ids=None, board_id=None): if board_id is None: board_id = self.board_id _, data = self._make_request(f"api/v2/boards/{board_id}/blocks?all=true") @@ -180,16 +281,22 @@ def get_cards(self, list_ids, board_id=None): view_id = [card_dict for card_dict in data if card_dict["type"] == "view"][0][ "id" ] - data = [ - card_dict - for card_dict in data - if card_dict["type"] == "card" - and card_dict["fields"]["properties"].get(list_prop, "") in list_ids - ] + if list_ids: + data = [ + card_dict + for card_dict in data + if card_dict["type"] == "card" + and card_dict["fields"]["properties"].get(list_prop, "") in list_ids + ] + else: + data = [ + card_dict + for card_dict in data + if card_dict["type"] == "card" + ] for card_dict in data: card = objects.TrelloCard.from_focalboard_dict(card_dict) card.url = urljoin(self.url, f"{board_id}/{view_id}/{card.id}") - print(card.url) # TODO: move this to app state for trello_list in lists: if trello_list.id == card_dict["fields"]["properties"].get( @@ -240,7 +347,9 @@ def _update_from_config(self): ) except Exception as e: # TODO remove this when main board is migrated - logger.error(f"something went wrong when setting up focalboard client", exc_info=e) + logger.error( + "something went wrong when setting up focalboard client", exc_info=e + ) pass def _make_request(self, uri, payload={}): diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py index d59f8c3..4575891 100644 --- a/src/jobs/__init__.py +++ b/src/jobs/__init__.py @@ -37,6 +37,7 @@ from .shrug_job import ShrugJob from .site_health_check_job import SiteHealthCheckJob from .tg_analytics_report_job import TgAnalyticsReportJob +from .board_state_job import BoardStateJob from .trello_board_state_job import TrelloBoardStateJob from .trello_board_state_notifications_job import TrelloBoardStateNotificationsJob from .trello_get_articles_arts_job import TrelloGetArticlesArtsJob diff --git a/src/jobs/board_state_job.py b/src/jobs/board_state_job.py new file mode 100644 index 0000000..d9f31d6 --- /dev/null +++ b/src/jobs/board_state_job.py @@ -0,0 +1,89 @@ +import datetime +import logging +from typing import Callable, List + +from ..app_context import AppContext +from ..consts import TrelloCardColor +from ..strings import load +from ..tg.sender import pretty_send +from ..trello.trello_objects import TrelloCard +from ..utils import card_checks_focalboard +from .base_job import BaseJob +from .utils import get_cards_by_curator, retrieve_usernames + +logger = logging.getLogger(__name__) + + +class BoardStateJob(BaseJob): + @staticmethod + def _execute( + app_context: AppContext, send: Callable[[str], None], called_from_handler=False + ): + paragraphs = [ + load("trello_board_state_job__intro") + ] # list of paragraph strings + curator_cards = get_cards_by_curator(app_context, focalboard=True) + for curator, curator_cards in curator_cards.items(): + curator_name, _ = curator + card_paragraphs = [] + curator_cards.sort(key=lambda c: c.due if c.due else datetime.datetime.min) + for card in curator_cards: + card_paragraph = BoardStateJob._format_card( + card, + card_checks_focalboard.make_card_failure_reasons(card, app_context), + app_context, + ) + if card_paragraph: + card_paragraphs.append(card_paragraph) + if card_paragraphs: + paragraphs.append(f"⭐️ Куратор: {curator_name}") + paragraphs += card_paragraphs + pretty_send(paragraphs, send) + + @staticmethod + def _format_card( + card: TrelloCard, failure_reasons: List[str], app_context: AppContext + ) -> str: + if not failure_reasons: + return None + + failure_reasons_formatted = ", ".join(failure_reasons) + labels = ( + load( + "trello_board_state_job__card_labels", + names=", ".join( + # We filter BLACK cards as this is an auxiliary label + label.name + for label in card.labels + if label.color != TrelloCardColor.BLACK + ), + ) + if card.labels + else "" + ) + + # Avoiding message overflow, strip explanations in () + list_name = card.lst.name + "(" + list_name = list_name[: list_name.find("(")].strip() + + members = ( + load( + "trello_board_state_job__card_members", + members=", ".join( + retrieve_usernames(card.members, app_context.db_client) + ), + curators="", + ) + if card.members + else "" + ) + + return load( + "trello_board_state_job__card_2", + failure_reasons=failure_reasons_formatted, + url=card.url, + name=card.name, + labels=labels, + list_name=list_name, + members=members, + ) diff --git a/src/jobs/fill_posts_list_focalboard_job.py b/src/jobs/fill_posts_list_focalboard_job.py index f37d390..a13892a 100644 --- a/src/jobs/fill_posts_list_focalboard_job.py +++ b/src/jobs/fill_posts_list_focalboard_job.py @@ -9,7 +9,6 @@ from ..sheets.sheets_objects import RegistryPost from ..strings import load from ..tg.sender import pretty_send -from ..trello.trello_client import TrelloClient from .base_job import BaseJob from .utils import check_trello_card, format_errors @@ -29,12 +28,9 @@ def _execute( registry_posts += FillPostsListFocalboardJob._retrieve_cards_for_registry( focalboard_client=app_context.focalboard_client, - trello_client=app_context.trello_client, list_aliases=(TrelloListAlias.PROOFREADING, TrelloListAlias.DONE), all_rubrics=all_rubrics, errors=errors, - show_due=True, - strict_archive_rules=True, ) if len(errors) == 0: @@ -58,7 +54,6 @@ def _execute( @staticmethod def _retrieve_cards_for_registry( focalboard_client: FocalboardClient, - trello_client: TrelloClient, list_aliases: List[str], errors: dict, all_rubrics: List, @@ -78,26 +73,62 @@ def _retrieve_cards_for_registry( registry_posts = [] for card in cards: - # label_names = [label.name for label in card.labels] - # is_main_post = load("common_trello_label__main_post") in label_names - # is_archive_post = load("common_trello_label__archive") in label_names - if not card: parse_failure_counter += 1 continue card_fields = focalboard_client.get_custom_fields(card.id) - logger.info(card_fields) - - # registry_posts.append( - # RegistryPost( - # card, - # card_fields, - # is_main_post, - # is_archive_post, - # all_rubrics, - # ) - # ) + + label_names = [label.name for label in card_fields._data] + card.labels = card_fields._data + is_main_post = load("common_trello_label__main_post") in label_names + is_archive_post = load("common_trello_label__archive") in label_names + + card_due = focalboard_client.get_card_due(card.id) + card.due = ( + datetime.datetime.fromtimestamp(card_due / 1000) if card_due else None + ) + + card_is_ok = check_trello_card( + card, + errors, + is_bad_title=( + card_fields.title is None + and card.lst.id + != focalboard_client.lists_config[TrelloListAlias.EDITED_NEXT_WEEK] + ), + is_bad_google_doc=card_fields.google_doc is None, + is_bad_authors=len(card_fields.authors) == 0, + is_bad_editors=len(card_fields.editors) == 0, + is_bad_cover=card_fields.cover is None and not is_archive_post, + is_bad_illustrators=( + len(card_fields.illustrators) == 0 + and need_illustrators + and not is_archive_post + ), + is_bad_due_date=card.due is None and show_due, + is_bad_label_names=len( + [ + label + for label in card.labels + if label.color != TrelloCardColor.BLACK + ] + ) + == 0, + ) + + if not card_is_ok: + continue + + registry_posts.append( + RegistryPost( + card, + card_fields, + is_main_post, + is_archive_post, + all_rubrics, + ) + ) if parse_failure_counter > 0: logger.error(f"Unparsed cards encountered: {parse_failure_counter}") diff --git a/src/jobs/fill_posts_list_job.py b/src/jobs/fill_posts_list_job.py index 414d7f2..2ed1bf0 100644 --- a/src/jobs/fill_posts_list_job.py +++ b/src/jobs/fill_posts_list_job.py @@ -26,7 +26,9 @@ def _execute( registry_posts += FillPostsListJob._retrieve_cards_for_registry( trello_client=app_context.trello_client, - list_aliases=(TrelloListAlias.PROOFREADING, TrelloListAlias.DONE), + list_aliases=[ + TrelloListAlias.PUBLISHED + ], all_rubrics=all_rubrics, errors=errors, show_due=True, diff --git a/src/jobs/utils.py b/src/jobs/utils.py index e920f54..c6e1251 100644 --- a/src/jobs/utils.py +++ b/src/jobs/utils.py @@ -271,8 +271,11 @@ def check_trello_card( return True -def get_cards_by_curator(app_context: AppContext): - cards = app_context.trello_client.get_cards() +def get_cards_by_curator(app_context: AppContext, focalboard=False): + if focalboard: + cards = app_context.focalboard_client.get_cards() + else: + cards = app_context.trello_client.get_cards() curator_cards = defaultdict(list) for card in cards: curators = get_curators_by_card(card, app_context.db_client) diff --git a/src/tg/handlers/get_board_credentials_handler.py b/src/tg/handlers/get_board_credentials_handler.py index c9a4ab0..7095261 100644 --- a/src/tg/handlers/get_board_credentials_handler.py +++ b/src/tg/handlers/get_board_credentials_handler.py @@ -17,19 +17,18 @@ def get_board_credentials(update: telegram.Update, tg_context): member for member in DBClient().get_all_members() if member.telegram == f"@{get_sender_username(update)}" ), None) - if member is None or not member.trello: + if member is None or not member.focalboard: logger.usage( - f'Trello not found for {get_sender_username(update)}, ID={get_sender_id(update)}' + f'Focalboard not found for {get_sender_username(update)}, ID={get_sender_id(update)}' ) reply(load('get_board_credentials_handler__not_found'), update) return try: - print(member.trello) with open('board_credentials.json', encoding="utf-8") as fin: try: board_json = json.loads(fin.read()) creds = next(( - cred for cred in board_json if cred["trelloUsername"] == member.trello + cred for cred in board_json if cred["focalboardUsername"] == member.focalboard ), None) if not creds: logger.usage( diff --git a/src/trello/trello_objects.py b/src/trello/trello_objects.py index fff74dd..997bef8 100644 --- a/src/trello/trello_objects.py +++ b/src/trello/trello_objects.py @@ -2,7 +2,7 @@ import logging from datetime import datetime -from ..consts import TrelloCardColor, TrelloCustomFieldTypes +from ..consts import BoardCardColor, TrelloCardColor, TrelloCustomFieldTypes logger = logging.getLogger(__name__) @@ -87,6 +87,18 @@ def from_dict(cls, data): logger.error(f"Bad board label json {data}: {e}") return label + @classmethod + def from_focalboard_dict(cls, data): + label = cls() + try: + label.id = data["id"] + label.name = html.escape(data["value"]) + label.color = data["color"] + except Exception as e: + label._ok = False + logger.error(f"Bad board label json {data}: {e}") + return label + def to_dict(self): return { "id": self.id, @@ -177,6 +189,22 @@ def from_dict(cls, data): logger.error(f"Bad card label json {data}: {e}") return label + @classmethod + def from_focalboard_dict(cls, data): + label = cls() + try: + label.id = data["id"] + label.name = html.escape(data["value"]) + label.color = None + try: + label.color = BoardCardColor(data["color"]) + except Exception: + label.color = BoardCardColor(BoardCardColor.UNKNOWN) + except Exception as e: + label._ok = False + logger.error(f"Bad card label json {data}: {e}") + return label + def to_dict(self): return { "id": self.id, @@ -235,11 +263,7 @@ def from_focalboard_dict(cls, data): try: card.id = data["id"] card.name = html.escape(data["title"]) - # card.labels = [TrelloCardLabel.from_dict(label) for label in data["labels"]] - # card.url = data["shortUrl"] - # card.due = ( - # datetime.strptime(data["due"], TIME_FORMAT) if data["due"] else None - # ) + # card.labels=[TrelloCardLabel.from_focalboard_dict(label) for label in data["labels"]] except Exception as e: card._ok = False logger.error(f"Bad card json {data}: {e}") diff --git a/src/utils/card_checks.py b/src/utils/card_checks.py index 7051a51..76a045a 100644 --- a/src/utils/card_checks.py +++ b/src/utils/card_checks.py @@ -57,6 +57,7 @@ def is_author_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, TrelloListAlias.TO_CHIEF_EDITOR, TrelloListAlias.PROOFREADING, TrelloListAlias.DONE, + TrelloListAlias.PUBLISHED, ) list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases) return card.lst.id in list_ids, {} @@ -75,6 +76,7 @@ def is_tag_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dic TrelloListAlias.TO_CHIEF_EDITOR, TrelloListAlias.PROOFREADING, TrelloListAlias.DONE, + TrelloListAlias.PUBLISHED, ) list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases) return card.lst.id in list_ids, {} @@ -89,6 +91,7 @@ def is_doc_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dic TrelloListAlias.TO_CHIEF_EDITOR, TrelloListAlias.PROOFREADING, TrelloListAlias.DONE, + TrelloListAlias.PUBLISHED, ) list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases) if card.lst.id not in list_ids: @@ -107,6 +110,7 @@ def has_no_doc_access(card: TrelloCard, app_context: AppContext) -> Tuple[bool, TrelloListAlias.TO_CHIEF_EDITOR, TrelloListAlias.PROOFREADING, TrelloListAlias.DONE, + TrelloListAlias.PUBLISHED, ) list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases) if card.lst.id not in list_ids: diff --git a/src/utils/card_checks_focalboard.py b/src/utils/card_checks_focalboard.py new file mode 100644 index 0000000..159692f --- /dev/null +++ b/src/utils/card_checks_focalboard.py @@ -0,0 +1,131 @@ +import datetime +from typing import Tuple + +from ..app_context import AppContext +from ..consts import TrelloCardColor, TrelloListAlias +from ..strings import load +from ..trello.trello_objects import TrelloCard + + +def make_card_failure_reasons(card: TrelloCard, app_context: AppContext): + """ + Returns card description with failure reasons, if any. + If card does not match any of FILTER_TO_FAILURE_REASON, returns None. + """ + failure_reasons = [] + for filter_func, reason_alias in FILTER_TO_FAILURE_REASON.items(): + is_failed, kwargs = filter_func(card, app_context) + if is_failed: + reason = load(reason_alias, **kwargs) + if reason and len(failure_reasons) > 0: + reason = reason[0].lower() + reason[1:] + failure_reasons.append(reason) + return failure_reasons + + +def is_deadline_missed(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + list_ids = app_context.focalboard_client.get_list_id_from_aliases( + [TrelloListAlias.IN_PROGRESS] + ) + is_missed = ( + card.lst.id in list_ids + and card.due is not None + and card.due.date() < datetime.datetime.now().date() + ) + return is_missed, {"date": card.due.strftime("%d.%m")} if is_missed else {} + + +def is_due_date_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + if card.due: + return False, {} + list_ids = app_context.focalboard_client.get_list_id_from_aliases( + [TrelloListAlias.IN_PROGRESS] + ) + return card.lst.id in list_ids, {} + + +def is_author_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + if card.members: + return False, {} + + list_aliases = ( + TrelloListAlias.IN_PROGRESS, + TrelloListAlias.TO_EDITOR, + TrelloListAlias.EDITED_NEXT_WEEK, + TrelloListAlias.TO_SEO_EDITOR, + TrelloListAlias.EDITED_SOMETIMES, + TrelloListAlias.TO_CHIEF_EDITOR, + TrelloListAlias.PROOFREADING, + TrelloListAlias.DONE, + ) + list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases) + return card.lst.id in list_ids, {} + + +def is_tag_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + if card.labels: + return False, {} + + list_aliases = ( + TrelloListAlias.IN_PROGRESS, + TrelloListAlias.TO_EDITOR, + TrelloListAlias.EDITED_NEXT_WEEK, + TrelloListAlias.TO_SEO_EDITOR, + TrelloListAlias.EDITED_SOMETIMES, + TrelloListAlias.TO_CHIEF_EDITOR, + TrelloListAlias.PROOFREADING, + TrelloListAlias.DONE, + ) + list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases) + return card.lst.id in list_ids, {} + + +def is_doc_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + list_aliases = ( + TrelloListAlias.TO_EDITOR, + TrelloListAlias.EDITED_NEXT_WEEK, + TrelloListAlias.TO_SEO_EDITOR, + TrelloListAlias.EDITED_SOMETIMES, + TrelloListAlias.TO_CHIEF_EDITOR, + TrelloListAlias.PROOFREADING, + TrelloListAlias.DONE, + ) + list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases) + if card.lst.id not in list_ids: + return False, {} + + doc_url = app_context.focalboard_client.get_custom_fields(card.id).google_doc + return not doc_url, {} + + +def has_no_doc_access(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]: + list_aliases = ( + TrelloListAlias.TO_EDITOR, + TrelloListAlias.EDITED_NEXT_WEEK, + TrelloListAlias.TO_SEO_EDITOR, + TrelloListAlias.EDITED_SOMETIMES, + TrelloListAlias.TO_CHIEF_EDITOR, + TrelloListAlias.PROOFREADING, + TrelloListAlias.DONE, + ) + list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases) + if card.lst.id not in list_ids: + return False, {} + + doc_url = app_context.focalboard_client.get_custom_fields(card.id).google_doc + if not doc_url: + # should be handled by is_doc_missing + return False, {} + + is_open_for_edit = app_context.drive_client.is_open_for_edit(doc_url) + return not is_open_for_edit, {} + + +FILTER_TO_FAILURE_REASON = { + is_author_missing: "trello_board_state_job__title_author_missing", + is_due_date_missing: "trello_board_state_job__title_due_date_missing", + is_deadline_missed: "trello_board_state_job__title_due_date_expired", + is_tag_missing: "trello_board_state_job__title_tag_missing", + is_doc_missing: "trello_board_state_job__title_no_doc", + has_no_doc_access: "trello_board_state_job__title_no_doc_access", +} diff --git a/src/utils/log_handler.py b/src/utils/log_handler.py index 281b80e..76b6127 100644 --- a/src/utils/log_handler.py +++ b/src/utils/log_handler.py @@ -1,8 +1,6 @@ import html from logging import ERROR, Formatter, LogRecord, StreamHandler -from sentry_sdk import capture_message - from ..consts import LOG_FORMAT, USAGE_LOG_LEVEL from ..tg.sender import TelegramSender from .singleton import Singleton @@ -49,21 +47,6 @@ def emit(self, record: LogRecord): error_message = f"{record.levelname} - {record.module} - {record.message}" if record.exc_text: error_message += f" - {record.exc_text}" - try: - capture_message(error_message) - except Exception as e: - # if it can't send a message, still should log it to the stream - super().emit( - LogRecord( - name=__name__, - level=ERROR, - pathname=None, - lineno=-1, - msg="Failed to capture sentry log", - args=None, - exc_info=e, - ) - ) try: self.tg_sender.send_error_log( f"{html.escape(error_message)}" diff --git a/src/utils/uptrace_logger.py b/src/utils/uptrace_logger.py new file mode 100644 index 0000000..4b20639 --- /dev/null +++ b/src/utils/uptrace_logger.py @@ -0,0 +1,36 @@ +import logging + +import grpc +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.resources import Resource + +from src.consts import APP_SOURCE + + +def add_uptrace_logging(dsn): + resource = Resource( + attributes={ + "service.name": "sysblokbot", + "service.version": "1.0.0", + "deployment.environment": APP_SOURCE, + } + ) + logger_provider = LoggerProvider(resource=resource) + set_logger_provider(logger_provider) + + exporter = OTLPLogExporter( + endpoint="otlp.uptrace.dev:4317", + headers=(("uptrace-dsn", dsn),), + timeout=5, + compression=grpc.Compression.Gzip, + ) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + + handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + + logging.getLogger().addHandler(handler)