diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml
index 7e159db..21a1a19 100644
--- a/.github/workflows/publish_dev.yml
+++ b/.github/workflows/publish_dev.yml
@@ -117,13 +117,14 @@ jobs:
ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/testing
TELEGRAM_TEST_TOKEN: ${{ secrets.TELEGRAM_TEST_TOKEN }}
TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }}
+ UPTRACE_DSN: ${{ secrets.UPTRACE_DSN }}
with:
host: ${{ secrets.MASTER_HOST }}
username: ${{ secrets.MASTER_DO_USER }}
passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }}
key: ${{ secrets.MASTER_DO_SSH_KEY }}
port: ${{ secrets.MASTER_PORT }}
- envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID
+ envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN
script: |
export CONTAINER_ID=$(docker ps -aq --filter name=testing)
export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:testing')
@@ -135,8 +136,9 @@ jobs:
touch ${{ env.ROOT_DIR }}/strings.sqlite
touch ${{ env.ROOT_DIR }}/board_credentials.json
docker run -dit --name sysblokbot-testing \
- --env APP_SOURCE="github CI" --restart unless-stopped \
+ --env APP_SOURCE="testing" --restart unless-stopped \
--env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \
+ --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \
--env TELEGRAM_TOKEN="${{ env.TELEGRAM_TEST_TOKEN }}" \
-v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \
-v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \
diff --git a/.github/workflows/publish_master.yml b/.github/workflows/publish_master.yml
index a80e1ca..777a24c 100644
--- a/.github/workflows/publish_master.yml
+++ b/.github/workflows/publish_master.yml
@@ -120,13 +120,14 @@ jobs:
ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/prod
TELEGRAM_TOKEN: ${{ secrets.TELEGRAM_PROD_TOKEN }}
TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }}
+ UPTRACE_DSN: ${{ secrets.UPTRACE_DSN }}
with:
host: ${{ secrets.MASTER_HOST }}
username: ${{ secrets.MASTER_DO_USER }}
passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }}
key: ${{ secrets.MASTER_DO_SSH_KEY }}
port: ${{ secrets.MASTER_PORT }}
- envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID
+ envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN
script: |
export CONTAINER_ID=$(docker ps -aq --filter name=prod)
export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:prod')
@@ -138,9 +139,10 @@ jobs:
touch ${{ env.ROOT_DIR }}/strings.sqlite
touch ${{ env.ROOT_DIR }}/board_credentials.json
docker run -dit --name sysblokbot-prod \
- --env APP_SOURCE="github CI" --restart unless-stopped \
+ --env APP_SOURCE="prod" --restart unless-stopped \
--env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \
--env TELEGRAM_TOKEN="${{ env.TELEGRAM_TOKEN }}" \
+ --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \
-v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \
-v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \
-v ${{ env.ROOT_DIR }}/sysblokbot.sqlite:/app/sysblokbot.sqlite \
diff --git a/.isort.cfg b/.isort.cfg
index d8a9dc2..75bd081 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -1,2 +1,2 @@
[settings]
-known_third_party = apiclient,bs4,conftest,dateutil,deepdiff,facebook,fakes,freezegun,googleapiclient,matplotlib,numpy,oauth2client,pytest,pytest_report,requests,schedule,sentry_sdk,setuptools,sheetfu,sqlalchemy,telegram,telethon,utils,vk_api
+known_third_party = apiclient,bs4,conftest,dateutil,deepdiff,facebook,fakes,freezegun,googleapiclient,matplotlib,numpy,oauth2client,pytest,pytest_report,requests,schedule,setuptools,sheetfu,sqlalchemy,telegram,telethon,utils,vk_api
diff --git a/Dockerfile b/Dockerfile
index 8256f5e..d9be45b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,8 +9,10 @@ COPY . /app
ARG COMMIT_HASH
ARG COMMIT_HASH_SHORT
+ARG UPTRACE_DSN
ENV COMMIT_HASH=$COMMIT_HASH
ENV COMMIT_HASH_SHORT=$COMMIT_HASH_SHORT
+ENV UPTRACE_DSN=$UPTRACE_DSN
ENV MUSL_LOCALE_DEPS cmake make musl-dev gcc gettext-dev libintl
diff --git a/app.py b/app.py
index 9dc94c2..204cda4 100644
--- a/app.py
+++ b/app.py
@@ -5,7 +5,6 @@
import logging
import requests
-import sentry_sdk
from src import consts
from src.bot import SysBlokBot
@@ -13,6 +12,7 @@
from src.scheduler import JobScheduler
from src.tg.sender import TelegramSender
from src.utils.log_handler import ErrorBroadcastHandler
+from src.utils.uptrace_logger import add_uptrace_logging
locale.setlocale(locale.LC_TIME, "ru_RU.UTF-8")
logging.basicConfig(format=consts.LOG_FORMAT, level=logging.INFO)
@@ -34,10 +34,6 @@ def get_bot():
if not config:
raise ValueError("Could not load config, can't go on")
- sentry_dsn = config.get("sentry_dsn", None)
- if sentry_dsn:
- sentry_sdk.init(dsn=sentry_dsn, traces_sample_rate=1.0)
-
scheduler = JobScheduler()
jobs_config_file_key = ConfigManager().get_jobs_config_file_key()
@@ -66,6 +62,8 @@ def get_bot():
for handler in logging.getLogger().handlers:
logging.getLogger().removeHandler(handler)
logging.getLogger().addHandler(ErrorBroadcastHandler(tg_sender))
+ if consts.UPTRACE_DSN:
+ add_uptrace_logging(consts.UPTRACE_DSN)
# Scheduler must be run after clients initialized
scheduler.run()
@@ -82,7 +80,6 @@ def get_bot():
def report_critical_error(e: BaseException):
- sentry_sdk.capture_exception(e)
requests.post(
url=f"https://api.telegram.org/bot{consts.TELEGRAM_TOKEN}/sendMessage",
json={
diff --git a/requirements.txt b/requirements.txt
index a20a2ee..bd2f1be 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,6 +17,7 @@ cryptography==3.1.1
cycler==0.11.0
decorator==4.4.2
deepdiff==4.3.2
+Deprecated==1.2.14
distlib==0.3.6
distro==1.4.0
-e git+https://github.com/mobolic/facebook-sdk.git@ffd9980700be48964d6a6a61144edb1c3ea29cff#egg=facebook_sdk
@@ -29,13 +30,14 @@ google-api-python-client==1.9.1
google-auth==1.16.0
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
-googleapis-common-protos==1.51.0
+googleapis-common-protos==1.63.0
+grpcio==1.63.0
gspread==3.4.2
html5lib==1.0.1
httplib2==0.18.1
identify==2.5.11
idna==2.9
-importlib-metadata==1.6.0
+importlib-metadata==7.0.0
ipaddr==2.2.0
isort==4.3.21
kiwisolver==1.4.4
@@ -50,6 +52,14 @@ nodeenv==1.7.0
numpy==1.24.1
oauth2client==4.1.3
oauthlib==3.1.0
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-exporter-otlp-proto-http==1.24.0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
ordered-set==3.1.1
packaging==20.3
pathspec==0.10.3
@@ -59,7 +69,7 @@ platformdirs==2.6.2
pluggy==0.13.1
pre-commit==2.21.0
progress==1.5
-protobuf==3.12.2
+protobuf==4.25.3
py==1.8.1
pyaes==1.6.1
pyasn1==0.4.8
@@ -81,7 +91,6 @@ retrying==1.3.3
rope==0.17.0
rsa==4.0
-e git+https://github.com/dbader/schedule.git@3eac646a8d2658929587d7454bd2c85696df254e#egg=schedule
-sentry-sdk==1.10.1
sheetfu==1.5.3
six==1.14.0
soupsieve==2.3.2.post1
diff --git a/src/bot.py b/src/bot.py
index e62c8ec..dc4065c 100644
--- a/src/bot.py
+++ b/src/bot.py
@@ -26,7 +26,7 @@
from .tg import handlers, sender
from .tg.handlers.utils import admin_only, direct_message_only, manager_only
-logging.addLevelName(USAGE_LOG_LEVEL, "USAGE")
+logging.addLevelName(USAGE_LOG_LEVEL, "NOTICE")
def usage(self, message, *args, **kws):
@@ -88,6 +88,12 @@ def init_handlers(self):
self.manager_reply_handler("trello_board_state_job"),
"получить сводку о состоянии доски",
)
+ self.add_manager_handler(
+ "get_board_state",
+ CommandCategories.SUMMARY,
+ self.manager_reply_handler("board_state_job"),
+ "получить сводку о состоянии доски (focalboard)",
+ )
self.add_manager_handler(
"get_editorial_board_stats",
CommandCategories.STATS,
diff --git a/src/consts.py b/src/consts.py
index 2d6e20d..81e3931 100644
--- a/src/consts.py
+++ b/src/consts.py
@@ -16,11 +16,13 @@
f'https://github.com/sysblok/sysblokbot/commit/{os.environ.get("COMMIT_HASH")}'
)
COMMIT_HASH = os.environ.get("COMMIT_HASH_SHORT")
+UPTRACE_DSN = os.environ.get("UPTRACE_DSN")
class AppSource(Enum):
DEFAULT = "manual"
- GITHUB = "github CI"
+ GITHUB = "prod"
+ GITHUB_DEV = "testing"
APP_SOURCE = os.environ.get("APP_SOURCE", AppSource.DEFAULT.value)
@@ -86,6 +88,19 @@ class TrelloCardColor(Enum):
UNKNOWN = "unknown"
+class BoardCardColor(Enum):
+ BLACK = "propColorGray"
+ BROWN = "propColorBrown"
+ ORANGE = "propColorOrange"
+ YELLOW = "propColorYellow"
+ GREEN = "propColorGreen"
+ BLUE = "propColorBlue"
+ PURPLE = "propColorPurple"
+ PINK = "propColorPink"
+ RED = "propColorRed"
+ UNKNOWN = "unknown"
+
+
class TrelloListAlias(Enum):
TOPIC_SUGGESTION = "trello_list_name__topic_suggestion"
TOPIC_READY = "trello_list_name__topic_ready"
@@ -97,6 +112,7 @@ class TrelloListAlias(Enum):
TO_CHIEF_EDITOR = "trello_list_name__to_chief_editor"
PROOFREADING = "trello_list_name__proofreading"
DONE = "trello_list_name__typesetting"
+ PUBLISHED = "trello_list_name__published"
BACK_BURNER = "trello_list_name__back_burner"
diff --git a/src/db/db_objects.py b/src/db/db_objects.py
index a4723ae..35d159f 100644
--- a/src/db/db_objects.py
+++ b/src/db/db_objects.py
@@ -110,6 +110,7 @@ class TeamMember(Base):
manager = Column(String)
telegram = Column(String)
trello = Column(String)
+ focalboard = Column(String)
roles = Column(String)
def __repr__(self):
@@ -125,6 +126,7 @@ def from_dict(cls, data):
member.manager = _get_str_data_item(data, "manager")
member.telegram = _get_str_data_item(data, "telegram")
member.trello = _get_str_data_item(data, "trello")
+ member.focalboard = _get_str_data_item(data, "focalboard")
return member
def to_dict(self):
@@ -136,6 +138,7 @@ def to_dict(self):
"manager": self.manager,
"telegram": self.telegram,
"trello": self.trello,
+ "focalboard": self.focalboard,
"roles": self.roles,
}
@@ -149,6 +152,7 @@ def from_sheetfu_item(cls, item):
member.manager = item.get_field_value(load("sheets__team__manager"))
member.telegram = item.get_field_value(load("sheets__team__telegram"))
member.trello = item.get_field_value(load("sheets__team__trello"))
+ member.focalboard = item.get_field_value(load("sheets__focalboard"))
return member
diff --git a/src/focalboard/focalboard_client.py b/src/focalboard/focalboard_client.py
index 16904d3..85d0f53 100644
--- a/src/focalboard/focalboard_client.py
+++ b/src/focalboard/focalboard_client.py
@@ -1,3 +1,4 @@
+import ast
import json
import logging
from typing import List
@@ -40,7 +41,7 @@ def get_lists(self, board_id=None, sorted=False):
for prop in [board for board in data if board["id"] == board_id][0][
"cardProperties"
]
- if prop["name"] == "List"
+ if prop["name"] == "Колонка"
][0]
lists_data = list_data["options"]
lists = [
@@ -50,8 +51,12 @@ def get_lists(self, board_id=None, sorted=False):
if sorted:
# we need to get sorting order from the view, which is currently not efficient
try:
- _, data = self._make_request(f"api/v2/boards/{board_id}/blocks?all=true")
- view = [card_dict for card_dict in data if card_dict["type"] == "view"][0]
+ _, data = self._make_request(
+ f"api/v2/boards/{board_id}/blocks?all=true"
+ )
+ view = [card_dict for card_dict in data if card_dict["type"] == "view"][
+ 0
+ ]
order = view["fields"]["visibleOptionIds"]
sorted_lists = []
for list_id in order:
@@ -59,7 +64,7 @@ def get_lists(self, board_id=None, sorted=False):
sorted_lists.append(this_list)
lists = sorted_lists
except Exception as e:
- logger.error(f"can't sort focalboard lists", exc_info=e)
+ logger.error("can't sort focalboard lists", exc_info=e)
logger.debug(f"get_lists: {lists}")
return lists
@@ -70,7 +75,7 @@ def get_list(self, board_id, list_id):
for prop in [board for board in data if board["id"] == board_id][0][
"cardProperties"
]
- if prop["name"] == "List"
+ if prop["name"] == "Колонка"
][0]["options"]
lst = [
objects.TrelloList.from_focalboard_dict(trello_list, board_id)
@@ -80,6 +85,25 @@ def get_list(self, board_id, list_id):
logger.debug(f"get_list: {lst}")
return lst
+ def _get_labels(self, board_id=None):
+ if board_id is None:
+ # default board
+ board_id = self.board_id
+ _, data = self._make_request("api/v2/teams/0/boards")
+ label_data = [
+ prop
+ for prop in [board for board in data if board["id"] == board_id][0][
+ "cardProperties"
+ ]
+ if prop["name"] == "Рубрика"
+ ][0]
+ labels_data = label_data["options"]
+ labels = [
+ objects.TrelloCardLabel.from_focalboard_dict(label)
+ for label in labels_data
+ ]
+ return labels
+
def _get_list_property(self, board_id):
_, data = self._make_request("api/v2/teams/0/boards")
return [
@@ -87,9 +111,47 @@ def _get_list_property(self, board_id):
for prop in [board for board in data if board["id"] == board_id][0][
"cardProperties"
]
- if prop["name"] == "List"
+ if prop["name"] == "Колонка"
+ ][0]["id"]
+
+ def _get_label_property(self, board_id=None):
+ if board_id is None:
+ # default board
+ board_id = self.board_id
+ _, data = self._make_request("api/v2/teams/0/boards")
+ return [
+ prop
+ for prop in [board for board in data if board["id"] == board_id][0][
+ "cardProperties"
+ ]
+ if prop["name"] == "Рубрика"
+ ][0]["id"]
+
+ def _get_due_property(self, board_id=None):
+ if board_id is None:
+ # default board
+ board_id = self.board_id
+ _, data = self._make_request("api/v2/teams/0/boards")
+ return [
+ prop
+ for prop in [board for board in data if board["id"] == board_id][0][
+ "cardProperties"
+ ]
+ if prop["name"] == "Дедлайн"
][0]["id"]
+ def get_card_due(self, card_id: str):
+ _, data = self._make_request(f"api/v2/cards/{card_id}")
+ due_id = self._get_due_property()
+ due_value = None
+
+ fields = data["properties"]
+ for type_id, value in fields.items():
+ if type_id == due_id:
+ due_value = value
+ due_value_dict = ast.literal_eval(due_value)
+ return due_value_dict.get("from", [])
+
def _get_member_property(self, board_id):
_, data = self._make_request("api/v2/teams/0/boards")
return [
@@ -97,7 +159,7 @@ def _get_member_property(self, board_id):
for prop in [board for board in data if board["id"] == board_id][0][
"cardProperties"
]
- if prop["name"] == "Assignee"
+ if prop["name"] == "Ответственный"
][0]["id"]
def get_list_id_from_aliases(self, list_aliases):
@@ -143,20 +205,59 @@ def get_board_custom_field_types(self):
logger.debug(f"get_board_custom_field_types: {custom_field_types}")
return custom_field_types
- def get_board_custom_fields_dict(self, card_id):
- custom_fields = self.get_board_custom_field_types()
- custom_fields_dict = {}
- for alias, type_id in self.custom_fields_config.items():
- suitable_fields = [fld for fld in custom_fields if fld.id == type_id]
- if len(suitable_fields) > 0:
- custom_fields_dict[alias] = suitable_fields[0]
- return custom_fields_dict
+ def get_username(self, user_id: str):
+ _, data = self._make_request(f"api/v2/users/{user_id}")
+ username = data["username"]
+ return username
def get_custom_fields(self, card_id: str) -> objects.CardCustomFields:
card_fields = objects.CardCustomFields(card_id)
- card_fields_dict = self.get_board_custom_fields_dict(card_id)
- card_fields._data = card_fields_dict
- logger.info(card_fields)
+ board_labels = self._get_labels()
+ card_fields_dict = {}
+ card_labels_ids = []
+ card_labels = []
+ _, data = self._make_request(f"api/v2/cards/{card_id}")
+ fields = data["properties"]
+ for alias, type_id in self.custom_fields_config.items():
+ if type_id in fields:
+ changed_alias = alias.name.split(".")[-1].lower()
+ card_fields_dict[changed_alias] = fields[type_id]
+
+ board_label_id = self._get_label_property()
+
+ for type_id, value in fields.items():
+ if type_id == board_label_id:
+ card_labels_ids = value
+
+ for card_label_id in card_labels_ids:
+ for board_label in board_labels:
+ if card_label_id == board_label.id:
+ card_labels.append(board_label)
+
+ card_fields.authors = [
+ author.strip()
+ for author in card_fields_dict.get("author", [])
+ ]
+ card_fields.editors = [
+ editor.strip()
+ for editor in card_fields_dict.get("editor", [])
+ ]
+ card_fields.illustrators = [
+ illustrator.strip()
+ for illustrator in card_fields_dict.get("illustrator", [])
+ ]
+ card_fields.cover = (
+ card_fields_dict["cover"] if "cover" in card_fields_dict else None
+ )
+ card_fields.google_doc = (
+ card_fields_dict["google_doc"] if "google_doc" in card_fields_dict else None
+ )
+ card_fields.title = (
+ card_fields_dict["title"] if "title" in card_fields_dict else None
+ )
+
+ card_fields._data = card_labels
+ return card_fields
def get_members(self, board_id) -> List[objects.TrelloMember]:
_, data = self._make_request(f"api/v2/boards/{board_id}/members")
@@ -167,7 +268,7 @@ def get_members(self, board_id) -> List[objects.TrelloMember]:
logger.debug(f"get_members: {members}")
return members
- def get_cards(self, list_ids, board_id=None):
+ def get_cards(self, list_ids=None, board_id=None):
if board_id is None:
board_id = self.board_id
_, data = self._make_request(f"api/v2/boards/{board_id}/blocks?all=true")
@@ -180,16 +281,22 @@ def get_cards(self, list_ids, board_id=None):
view_id = [card_dict for card_dict in data if card_dict["type"] == "view"][0][
"id"
]
- data = [
- card_dict
- for card_dict in data
- if card_dict["type"] == "card"
- and card_dict["fields"]["properties"].get(list_prop, "") in list_ids
- ]
+ if list_ids:
+ data = [
+ card_dict
+ for card_dict in data
+ if card_dict["type"] == "card"
+ and card_dict["fields"]["properties"].get(list_prop, "") in list_ids
+ ]
+ else:
+ data = [
+ card_dict
+ for card_dict in data
+ if card_dict["type"] == "card"
+ ]
for card_dict in data:
card = objects.TrelloCard.from_focalboard_dict(card_dict)
card.url = urljoin(self.url, f"{board_id}/{view_id}/{card.id}")
- print(card.url)
# TODO: move this to app state
for trello_list in lists:
if trello_list.id == card_dict["fields"]["properties"].get(
@@ -240,7 +347,9 @@ def _update_from_config(self):
)
except Exception as e:
# TODO remove this when main board is migrated
- logger.error(f"something went wrong when setting up focalboard client", exc_info=e)
+ logger.error(
+ "something went wrong when setting up focalboard client", exc_info=e
+ )
pass
def _make_request(self, uri, payload={}):
diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py
index d59f8c3..4575891 100644
--- a/src/jobs/__init__.py
+++ b/src/jobs/__init__.py
@@ -37,6 +37,7 @@
from .shrug_job import ShrugJob
from .site_health_check_job import SiteHealthCheckJob
from .tg_analytics_report_job import TgAnalyticsReportJob
+from .board_state_job import BoardStateJob
from .trello_board_state_job import TrelloBoardStateJob
from .trello_board_state_notifications_job import TrelloBoardStateNotificationsJob
from .trello_get_articles_arts_job import TrelloGetArticlesArtsJob
diff --git a/src/jobs/board_state_job.py b/src/jobs/board_state_job.py
new file mode 100644
index 0000000..d9f31d6
--- /dev/null
+++ b/src/jobs/board_state_job.py
@@ -0,0 +1,89 @@
+import datetime
+import logging
+from typing import Callable, List
+
+from ..app_context import AppContext
+from ..consts import TrelloCardColor
+from ..strings import load
+from ..tg.sender import pretty_send
+from ..trello.trello_objects import TrelloCard
+from ..utils import card_checks_focalboard
+from .base_job import BaseJob
+from .utils import get_cards_by_curator, retrieve_usernames
+
+logger = logging.getLogger(__name__)
+
+
+class BoardStateJob(BaseJob):
+ @staticmethod
+ def _execute(
+ app_context: AppContext, send: Callable[[str], None], called_from_handler=False
+ ):
+ paragraphs = [
+ load("trello_board_state_job__intro")
+ ] # list of paragraph strings
+ curator_cards = get_cards_by_curator(app_context, focalboard=True)
+ for curator, curator_cards in curator_cards.items():
+ curator_name, _ = curator
+ card_paragraphs = []
+ curator_cards.sort(key=lambda c: c.due if c.due else datetime.datetime.min)
+ for card in curator_cards:
+ card_paragraph = BoardStateJob._format_card(
+ card,
+ card_checks_focalboard.make_card_failure_reasons(card, app_context),
+ app_context,
+ )
+ if card_paragraph:
+ card_paragraphs.append(card_paragraph)
+ if card_paragraphs:
+ paragraphs.append(f"⭐️ Куратор: {curator_name}")
+ paragraphs += card_paragraphs
+ pretty_send(paragraphs, send)
+
+ @staticmethod
+ def _format_card(
+ card: TrelloCard, failure_reasons: List[str], app_context: AppContext
+ ) -> str:
+ if not failure_reasons:
+ return None
+
+ failure_reasons_formatted = ", ".join(failure_reasons)
+ labels = (
+ load(
+ "trello_board_state_job__card_labels",
+ names=", ".join(
+ # We filter BLACK cards as this is an auxiliary label
+ label.name
+ for label in card.labels
+ if label.color != TrelloCardColor.BLACK
+ ),
+ )
+ if card.labels
+ else ""
+ )
+
+ # Avoiding message overflow, strip explanations in ()
+ list_name = card.lst.name + "("
+ list_name = list_name[: list_name.find("(")].strip()
+
+ members = (
+ load(
+ "trello_board_state_job__card_members",
+ members=", ".join(
+ retrieve_usernames(card.members, app_context.db_client)
+ ),
+ curators="",
+ )
+ if card.members
+ else ""
+ )
+
+ return load(
+ "trello_board_state_job__card_2",
+ failure_reasons=failure_reasons_formatted,
+ url=card.url,
+ name=card.name,
+ labels=labels,
+ list_name=list_name,
+ members=members,
+ )
diff --git a/src/jobs/fill_posts_list_focalboard_job.py b/src/jobs/fill_posts_list_focalboard_job.py
index f37d390..a13892a 100644
--- a/src/jobs/fill_posts_list_focalboard_job.py
+++ b/src/jobs/fill_posts_list_focalboard_job.py
@@ -9,7 +9,6 @@
from ..sheets.sheets_objects import RegistryPost
from ..strings import load
from ..tg.sender import pretty_send
-from ..trello.trello_client import TrelloClient
from .base_job import BaseJob
from .utils import check_trello_card, format_errors
@@ -29,12 +28,9 @@ def _execute(
registry_posts += FillPostsListFocalboardJob._retrieve_cards_for_registry(
focalboard_client=app_context.focalboard_client,
- trello_client=app_context.trello_client,
list_aliases=(TrelloListAlias.PROOFREADING, TrelloListAlias.DONE),
all_rubrics=all_rubrics,
errors=errors,
- show_due=True,
- strict_archive_rules=True,
)
if len(errors) == 0:
@@ -58,7 +54,6 @@ def _execute(
@staticmethod
def _retrieve_cards_for_registry(
focalboard_client: FocalboardClient,
- trello_client: TrelloClient,
list_aliases: List[str],
errors: dict,
all_rubrics: List,
@@ -78,26 +73,62 @@ def _retrieve_cards_for_registry(
registry_posts = []
for card in cards:
- # label_names = [label.name for label in card.labels]
- # is_main_post = load("common_trello_label__main_post") in label_names
- # is_archive_post = load("common_trello_label__archive") in label_names
-
if not card:
parse_failure_counter += 1
continue
card_fields = focalboard_client.get_custom_fields(card.id)
- logger.info(card_fields)
-
- # registry_posts.append(
- # RegistryPost(
- # card,
- # card_fields,
- # is_main_post,
- # is_archive_post,
- # all_rubrics,
- # )
- # )
+
+ label_names = [label.name for label in card_fields._data]
+ card.labels = card_fields._data
+ is_main_post = load("common_trello_label__main_post") in label_names
+ is_archive_post = load("common_trello_label__archive") in label_names
+
+ card_due = focalboard_client.get_card_due(card.id)
+ card.due = (
+ datetime.datetime.fromtimestamp(card_due / 1000) if card_due else None
+ )
+
+ card_is_ok = check_trello_card(
+ card,
+ errors,
+ is_bad_title=(
+ card_fields.title is None
+ and card.lst.id
+ != focalboard_client.lists_config[TrelloListAlias.EDITED_NEXT_WEEK]
+ ),
+ is_bad_google_doc=card_fields.google_doc is None,
+ is_bad_authors=len(card_fields.authors) == 0,
+ is_bad_editors=len(card_fields.editors) == 0,
+ is_bad_cover=card_fields.cover is None and not is_archive_post,
+ is_bad_illustrators=(
+ len(card_fields.illustrators) == 0
+ and need_illustrators
+ and not is_archive_post
+ ),
+ is_bad_due_date=card.due is None and show_due,
+ is_bad_label_names=len(
+ [
+ label
+ for label in card.labels
+ if label.color != TrelloCardColor.BLACK
+ ]
+ )
+ == 0,
+ )
+
+ if not card_is_ok:
+ continue
+
+ registry_posts.append(
+ RegistryPost(
+ card,
+ card_fields,
+ is_main_post,
+ is_archive_post,
+ all_rubrics,
+ )
+ )
if parse_failure_counter > 0:
logger.error(f"Unparsed cards encountered: {parse_failure_counter}")
diff --git a/src/jobs/fill_posts_list_job.py b/src/jobs/fill_posts_list_job.py
index 414d7f2..2ed1bf0 100644
--- a/src/jobs/fill_posts_list_job.py
+++ b/src/jobs/fill_posts_list_job.py
@@ -26,7 +26,9 @@ def _execute(
registry_posts += FillPostsListJob._retrieve_cards_for_registry(
trello_client=app_context.trello_client,
- list_aliases=(TrelloListAlias.PROOFREADING, TrelloListAlias.DONE),
+ list_aliases=[
+ TrelloListAlias.PUBLISHED
+ ],
all_rubrics=all_rubrics,
errors=errors,
show_due=True,
diff --git a/src/jobs/utils.py b/src/jobs/utils.py
index e920f54..c6e1251 100644
--- a/src/jobs/utils.py
+++ b/src/jobs/utils.py
@@ -271,8 +271,11 @@ def check_trello_card(
return True
-def get_cards_by_curator(app_context: AppContext):
- cards = app_context.trello_client.get_cards()
+def get_cards_by_curator(app_context: AppContext, focalboard=False):
+ if focalboard:
+ cards = app_context.focalboard_client.get_cards()
+ else:
+ cards = app_context.trello_client.get_cards()
curator_cards = defaultdict(list)
for card in cards:
curators = get_curators_by_card(card, app_context.db_client)
diff --git a/src/tg/handlers/get_board_credentials_handler.py b/src/tg/handlers/get_board_credentials_handler.py
index c9a4ab0..7095261 100644
--- a/src/tg/handlers/get_board_credentials_handler.py
+++ b/src/tg/handlers/get_board_credentials_handler.py
@@ -17,19 +17,18 @@ def get_board_credentials(update: telegram.Update, tg_context):
member for member in DBClient().get_all_members()
if member.telegram == f"@{get_sender_username(update)}"
), None)
- if member is None or not member.trello:
+ if member is None or not member.focalboard:
logger.usage(
- f'Trello not found for {get_sender_username(update)}, ID={get_sender_id(update)}'
+ f'Focalboard not found for {get_sender_username(update)}, ID={get_sender_id(update)}'
)
reply(load('get_board_credentials_handler__not_found'), update)
return
try:
- print(member.trello)
with open('board_credentials.json', encoding="utf-8") as fin:
try:
board_json = json.loads(fin.read())
creds = next((
- cred for cred in board_json if cred["trelloUsername"] == member.trello
+ cred for cred in board_json if cred["focalboardUsername"] == member.focalboard
), None)
if not creds:
logger.usage(
diff --git a/src/trello/trello_objects.py b/src/trello/trello_objects.py
index fff74dd..997bef8 100644
--- a/src/trello/trello_objects.py
+++ b/src/trello/trello_objects.py
@@ -2,7 +2,7 @@
import logging
from datetime import datetime
-from ..consts import TrelloCardColor, TrelloCustomFieldTypes
+from ..consts import BoardCardColor, TrelloCardColor, TrelloCustomFieldTypes
logger = logging.getLogger(__name__)
@@ -87,6 +87,18 @@ def from_dict(cls, data):
logger.error(f"Bad board label json {data}: {e}")
return label
+ @classmethod
+ def from_focalboard_dict(cls, data):
+ label = cls()
+ try:
+ label.id = data["id"]
+ label.name = html.escape(data["value"])
+ label.color = data["color"]
+ except Exception as e:
+ label._ok = False
+ logger.error(f"Bad board label json {data}: {e}")
+ return label
+
def to_dict(self):
return {
"id": self.id,
@@ -177,6 +189,22 @@ def from_dict(cls, data):
logger.error(f"Bad card label json {data}: {e}")
return label
+ @classmethod
+ def from_focalboard_dict(cls, data):
+ label = cls()
+ try:
+ label.id = data["id"]
+ label.name = html.escape(data["value"])
+ label.color = None
+ try:
+ label.color = BoardCardColor(data["color"])
+ except Exception:
+ label.color = BoardCardColor(BoardCardColor.UNKNOWN)
+ except Exception as e:
+ label._ok = False
+ logger.error(f"Bad card label json {data}: {e}")
+ return label
+
def to_dict(self):
return {
"id": self.id,
@@ -235,11 +263,7 @@ def from_focalboard_dict(cls, data):
try:
card.id = data["id"]
card.name = html.escape(data["title"])
- # card.labels = [TrelloCardLabel.from_dict(label) for label in data["labels"]]
- # card.url = data["shortUrl"]
- # card.due = (
- # datetime.strptime(data["due"], TIME_FORMAT) if data["due"] else None
- # )
+ # card.labels=[TrelloCardLabel.from_focalboard_dict(label) for label in data["labels"]]
except Exception as e:
card._ok = False
logger.error(f"Bad card json {data}: {e}")
diff --git a/src/utils/card_checks.py b/src/utils/card_checks.py
index 7051a51..76a045a 100644
--- a/src/utils/card_checks.py
+++ b/src/utils/card_checks.py
@@ -57,6 +57,7 @@ def is_author_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool,
TrelloListAlias.TO_CHIEF_EDITOR,
TrelloListAlias.PROOFREADING,
TrelloListAlias.DONE,
+ TrelloListAlias.PUBLISHED,
)
list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases)
return card.lst.id in list_ids, {}
@@ -75,6 +76,7 @@ def is_tag_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dic
TrelloListAlias.TO_CHIEF_EDITOR,
TrelloListAlias.PROOFREADING,
TrelloListAlias.DONE,
+ TrelloListAlias.PUBLISHED,
)
list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases)
return card.lst.id in list_ids, {}
@@ -89,6 +91,7 @@ def is_doc_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dic
TrelloListAlias.TO_CHIEF_EDITOR,
TrelloListAlias.PROOFREADING,
TrelloListAlias.DONE,
+ TrelloListAlias.PUBLISHED,
)
list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases)
if card.lst.id not in list_ids:
@@ -107,6 +110,7 @@ def has_no_doc_access(card: TrelloCard, app_context: AppContext) -> Tuple[bool,
TrelloListAlias.TO_CHIEF_EDITOR,
TrelloListAlias.PROOFREADING,
TrelloListAlias.DONE,
+ TrelloListAlias.PUBLISHED,
)
list_ids = app_context.trello_client.get_list_id_from_aliases(list_aliases)
if card.lst.id not in list_ids:
diff --git a/src/utils/card_checks_focalboard.py b/src/utils/card_checks_focalboard.py
new file mode 100644
index 0000000..159692f
--- /dev/null
+++ b/src/utils/card_checks_focalboard.py
@@ -0,0 +1,131 @@
+import datetime
+from typing import Tuple
+
+from ..app_context import AppContext
+from ..consts import TrelloCardColor, TrelloListAlias
+from ..strings import load
+from ..trello.trello_objects import TrelloCard
+
+
+def make_card_failure_reasons(card: TrelloCard, app_context: AppContext):
+ """
+ Returns card description with failure reasons, if any.
+ If card does not match any of FILTER_TO_FAILURE_REASON, returns None.
+ """
+ failure_reasons = []
+ for filter_func, reason_alias in FILTER_TO_FAILURE_REASON.items():
+ is_failed, kwargs = filter_func(card, app_context)
+ if is_failed:
+ reason = load(reason_alias, **kwargs)
+ if reason and len(failure_reasons) > 0:
+ reason = reason[0].lower() + reason[1:]
+ failure_reasons.append(reason)
+ return failure_reasons
+
+
+def is_deadline_missed(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(
+ [TrelloListAlias.IN_PROGRESS]
+ )
+ is_missed = (
+ card.lst.id in list_ids
+ and card.due is not None
+ and card.due.date() < datetime.datetime.now().date()
+ )
+ return is_missed, {"date": card.due.strftime("%d.%m")} if is_missed else {}
+
+
+def is_due_date_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ if card.due:
+ return False, {}
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(
+ [TrelloListAlias.IN_PROGRESS]
+ )
+ return card.lst.id in list_ids, {}
+
+
+def is_author_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ if card.members:
+ return False, {}
+
+ list_aliases = (
+ TrelloListAlias.IN_PROGRESS,
+ TrelloListAlias.TO_EDITOR,
+ TrelloListAlias.EDITED_NEXT_WEEK,
+ TrelloListAlias.TO_SEO_EDITOR,
+ TrelloListAlias.EDITED_SOMETIMES,
+ TrelloListAlias.TO_CHIEF_EDITOR,
+ TrelloListAlias.PROOFREADING,
+ TrelloListAlias.DONE,
+ )
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases)
+ return card.lst.id in list_ids, {}
+
+
+def is_tag_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ if card.labels:
+ return False, {}
+
+ list_aliases = (
+ TrelloListAlias.IN_PROGRESS,
+ TrelloListAlias.TO_EDITOR,
+ TrelloListAlias.EDITED_NEXT_WEEK,
+ TrelloListAlias.TO_SEO_EDITOR,
+ TrelloListAlias.EDITED_SOMETIMES,
+ TrelloListAlias.TO_CHIEF_EDITOR,
+ TrelloListAlias.PROOFREADING,
+ TrelloListAlias.DONE,
+ )
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases)
+ return card.lst.id in list_ids, {}
+
+
+def is_doc_missing(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ list_aliases = (
+ TrelloListAlias.TO_EDITOR,
+ TrelloListAlias.EDITED_NEXT_WEEK,
+ TrelloListAlias.TO_SEO_EDITOR,
+ TrelloListAlias.EDITED_SOMETIMES,
+ TrelloListAlias.TO_CHIEF_EDITOR,
+ TrelloListAlias.PROOFREADING,
+ TrelloListAlias.DONE,
+ )
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases)
+ if card.lst.id not in list_ids:
+ return False, {}
+
+ doc_url = app_context.focalboard_client.get_custom_fields(card.id).google_doc
+ return not doc_url, {}
+
+
+def has_no_doc_access(card: TrelloCard, app_context: AppContext) -> Tuple[bool, dict]:
+ list_aliases = (
+ TrelloListAlias.TO_EDITOR,
+ TrelloListAlias.EDITED_NEXT_WEEK,
+ TrelloListAlias.TO_SEO_EDITOR,
+ TrelloListAlias.EDITED_SOMETIMES,
+ TrelloListAlias.TO_CHIEF_EDITOR,
+ TrelloListAlias.PROOFREADING,
+ TrelloListAlias.DONE,
+ )
+ list_ids = app_context.focalboard_client.get_list_id_from_aliases(list_aliases)
+ if card.lst.id not in list_ids:
+ return False, {}
+
+ doc_url = app_context.focalboard_client.get_custom_fields(card.id).google_doc
+ if not doc_url:
+ # should be handled by is_doc_missing
+ return False, {}
+
+ is_open_for_edit = app_context.drive_client.is_open_for_edit(doc_url)
+ return not is_open_for_edit, {}
+
+
+FILTER_TO_FAILURE_REASON = {
+ is_author_missing: "trello_board_state_job__title_author_missing",
+ is_due_date_missing: "trello_board_state_job__title_due_date_missing",
+ is_deadline_missed: "trello_board_state_job__title_due_date_expired",
+ is_tag_missing: "trello_board_state_job__title_tag_missing",
+ is_doc_missing: "trello_board_state_job__title_no_doc",
+ has_no_doc_access: "trello_board_state_job__title_no_doc_access",
+}
diff --git a/src/utils/log_handler.py b/src/utils/log_handler.py
index 281b80e..76b6127 100644
--- a/src/utils/log_handler.py
+++ b/src/utils/log_handler.py
@@ -1,8 +1,6 @@
import html
from logging import ERROR, Formatter, LogRecord, StreamHandler
-from sentry_sdk import capture_message
-
from ..consts import LOG_FORMAT, USAGE_LOG_LEVEL
from ..tg.sender import TelegramSender
from .singleton import Singleton
@@ -49,21 +47,6 @@ def emit(self, record: LogRecord):
error_message = f"{record.levelname} - {record.module} - {record.message}"
if record.exc_text:
error_message += f" - {record.exc_text}"
- try:
- capture_message(error_message)
- except Exception as e:
- # if it can't send a message, still should log it to the stream
- super().emit(
- LogRecord(
- name=__name__,
- level=ERROR,
- pathname=None,
- lineno=-1,
- msg="Failed to capture sentry log",
- args=None,
- exc_info=e,
- )
- )
try:
self.tg_sender.send_error_log(
f"{html.escape(error_message)}
"
diff --git a/src/utils/uptrace_logger.py b/src/utils/uptrace_logger.py
new file mode 100644
index 0000000..4b20639
--- /dev/null
+++ b/src/utils/uptrace_logger.py
@@ -0,0 +1,36 @@
+import logging
+
+import grpc
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
+ OTLPLogExporter,
+)
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+
+from src.consts import APP_SOURCE
+
+
+def add_uptrace_logging(dsn):
+ resource = Resource(
+ attributes={
+ "service.name": "sysblokbot",
+ "service.version": "1.0.0",
+ "deployment.environment": APP_SOURCE,
+ }
+ )
+ logger_provider = LoggerProvider(resource=resource)
+ set_logger_provider(logger_provider)
+
+ exporter = OTLPLogExporter(
+ endpoint="otlp.uptrace.dev:4317",
+ headers=(("uptrace-dsn", dsn),),
+ timeout=5,
+ compression=grpc.Compression.Gzip,
+ )
+ logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+ handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
+
+ logging.getLogger().addHandler(handler)