feat: uptrace monitoring

.github/workflows/publish_dev.yml

@@ -117,13 +117,14 @@ jobs:
           ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/testing
           TELEGRAM_TEST_TOKEN: ${{ secrets.TELEGRAM_TEST_TOKEN }}
           TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }}
+          UPTRACE_DSN: ${{ UPTRACE_DSN }}
         with:
           host: ${{ secrets.MASTER_HOST }}
           username: ${{ secrets.MASTER_DO_USER }}
           passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }}
           key: ${{ secrets.MASTER_DO_SSH_KEY }}
           port: ${{ secrets.MASTER_PORT }}
-          envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID
+          envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TEST_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN
           script: |
             export CONTAINER_ID=$(docker ps -aq --filter name=testing)
             export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:testing')
@@ -135,8 +136,9 @@ jobs:
             touch ${{ env.ROOT_DIR }}/strings.sqlite
             touch ${{ env.ROOT_DIR }}/board_credentials.json
             docker run -dit --name sysblokbot-testing \
-              --env APP_SOURCE="github CI" --restart unless-stopped \
+              --env APP_SOURCE="testing" --restart unless-stopped \
               --env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \
+              --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \
               --env TELEGRAM_TOKEN="${{ env.TELEGRAM_TEST_TOKEN }}" \
               -v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \
               -v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \

.github/workflows/publish_master.yml

@@ -120,13 +120,14 @@ jobs:
           ROOT_DIR: /home/${{ secrets.MASTER_DO_USER }}/prod
           TELEGRAM_TOKEN: ${{ secrets.TELEGRAM_PROD_TOKEN }}
           TELEGRAM_ERROR_CHAT_ID: ${{ secrets.TELEGRAM_ERROR_CHAT_ID }}
+          UPTRACE_DSN: ${{ UPTRACE_DSN }}
         with:
           host: ${{ secrets.MASTER_HOST }}
           username: ${{ secrets.MASTER_DO_USER }}
           passphrase: ${{ secrets.MASTER_DO_SSH_KEY_PASSWORD }}
           key: ${{ secrets.MASTER_DO_SSH_KEY }}
           port: ${{ secrets.MASTER_PORT }}
-          envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID
+          envs: GITHUB_USERNAME, GITHUB_TOKEN, IMAGE_NAME, ROOT_DIR, TELEGRAM_TOKEN, TELEGRAM_ERROR_CHAT_ID, UPTRACE_DSN
           script: |
             export CONTAINER_ID=$(docker ps -aq --filter name=prod)
             export IMAGE_ID=$(docker images -aq --filter reference='docker.pkg.github.com/sysblok/sysblokbot/sysblokbot:prod')
@@ -138,9 +139,10 @@ jobs:
             touch ${{ env.ROOT_DIR }}/strings.sqlite
             touch ${{ env.ROOT_DIR }}/board_credentials.json
             docker run -dit --name sysblokbot-prod \
-              --env APP_SOURCE="github CI" --restart unless-stopped \
+              --env APP_SOURCE="prod" --restart unless-stopped \
               --env TELEGRAM_ERROR_CHAT_ID="${{ env.TELEGRAM_ERROR_CHAT_ID }}" \
               --env TELEGRAM_TOKEN="${{ env.TELEGRAM_TOKEN }}" \
+              --env UPTRACE_DSN="${{ env.UPTRACE_DSN }}" \
               -v ${{ env.ROOT_DIR }}/config_override.json:/app/config_override.json \
               -v ${{ env.ROOT_DIR }}/config_gs.json:/app/config_gs.json \
               -v ${{ env.ROOT_DIR }}/sysblokbot.sqlite:/app/sysblokbot.sqlite \

Dockerfile

@@ -9,8 +9,10 @@ COPY . /app
 
 ARG COMMIT_HASH
 ARG COMMIT_HASH_SHORT
+ARG UPTRACE_DSN
 ENV COMMIT_HASH=$COMMIT_HASH
 ENV COMMIT_HASH_SHORT=$COMMIT_HASH_SHORT
+ENV UPTRACE_DSN=$UPTRACE_DSN
 
 
 ENV MUSL_LOCALE_DEPS cmake make musl-dev gcc gettext-dev libintl

app.py

@@ -12,6 +12,7 @@
 from src.scheduler import JobScheduler
 from src.tg.sender import TelegramSender
 from src.utils.log_handler import ErrorBroadcastHandler
+from src.utils.uptrace_logger import add_uptrace_logging
 
 locale.setlocale(locale.LC_TIME, "ru_RU.UTF-8")
 logging.basicConfig(format=consts.LOG_FORMAT, level=logging.INFO)
@@ -61,6 +62,8 @@ def get_bot():
     for handler in logging.getLogger().handlers:
         logging.getLogger().removeHandler(handler)
     logging.getLogger().addHandler(ErrorBroadcastHandler(tg_sender))
+    if consts.UPTRACE_DSN:
+        add_uptrace_logging(consts.UPTRACE_DSN)
 
     # Scheduler must be run after clients initialized
     scheduler.run()

requirements.txt

@@ -17,6 +17,7 @@ cryptography==3.1.1
 cycler==0.11.0
 decorator==4.4.2
 deepdiff==4.3.2
+Deprecated==1.2.14
 distlib==0.3.6
 distro==1.4.0
 -e git+https://github.com/mobolic/facebook-sdk.git@ffd9980700be48964d6a6a61144edb1c3ea29cff#egg=facebook_sdk
@@ -29,13 +30,14 @@ google-api-python-client==1.9.1
 google-auth==1.16.0
 google-auth-httplib2==0.0.3
 google-auth-oauthlib==0.4.1
-googleapis-common-protos==1.51.0
+googleapis-common-protos==1.63.0
+grpcio==1.63.0
 gspread==3.4.2
 html5lib==1.0.1
 httplib2==0.18.1
 identify==2.5.11
 idna==2.9
-importlib-metadata==1.6.0
+importlib-metadata==7.0.0
 ipaddr==2.2.0
 isort==4.3.21
 kiwisolver==1.4.4
@@ -50,6 +52,14 @@ nodeenv==1.7.0
 numpy==1.24.1
 oauth2client==4.1.3
 oauthlib==3.1.0
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-exporter-otlp-proto-http==1.24.0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
 ordered-set==3.1.1
 packaging==20.3
 pathspec==0.10.3
@@ -59,7 +69,7 @@ platformdirs==2.6.2
 pluggy==0.13.1
 pre-commit==2.21.0
 progress==1.5
-protobuf==3.12.2
+protobuf==4.25.3
 py==1.8.1
 pyaes==1.6.1
 pyasn1==0.4.8

src/consts.py

@@ -16,11 +16,13 @@
     f'https://github.com/sysblok/sysblokbot/commit/{os.environ.get("COMMIT_HASH")}'
 )
 COMMIT_HASH = os.environ.get("COMMIT_HASH_SHORT")
+UPTRACE_DSN = os.environ.get("UPTRACE_DSN")
 
 
 class AppSource(Enum):
     DEFAULT = "manual"
-    GITHUB = "github CI"
+    GITHUB = "prod"
+    GITHUB_DEV = "testing"
 
 
 APP_SOURCE = os.environ.get("APP_SOURCE", AppSource.DEFAULT.value)

src/utils/uptrace_logger.py

@@ -0,0 +1,36 @@
+import logging
+
+import grpc
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import (
+    OTLPLogExporter,
+)
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.resources import Resource
+
+from src.consts import APP_SOURCE
+
+
+def add_uptrace_logging(dsn):
+    resource = Resource(
+        attributes={
+            "service.name": "sysblokbot",
+            "service.version": "1.0.0",
+            "deployment.environment": APP_SOURCE,
+        }
+    )
+    logger_provider = LoggerProvider(resource=resource)
+    set_logger_provider(logger_provider)
+
+    exporter = OTLPLogExporter(
+        endpoint="otlp.uptrace.dev:4317",
+        headers=(("uptrace-dsn", dsn),),
+        timeout=5,
+        compression=grpc.Compression.Gzip,
+    )
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+    handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
+
+    logging.getLogger().addHandler(handler)