feat: DIA-1306: improve logging (#174)

Dockerfile.app

@@ -71,5 +71,7 @@ FROM python-base as production
 # Copy artifacts from other stages
 COPY --from=venv-builder /usr/src/app /usr/src/app
 
+ENV LITELLM_LOG=WARNING
+
 # Set the working directory in the container to where the app will be run from
-WORKDIR /usr/src/app/server
+WORKDIR /usr/src/app/server

docker-compose.yml

@@ -33,6 +33,7 @@ services:
     environment:
       - REDIS_URL=redis://redis:6379/0
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9093
+      - LOG_LEVEL=DEBUG
     command:
       ["poetry", "run", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
   worker:
@@ -46,6 +47,7 @@ services:
       - REDIS_URL=redis://redis:6379/0
       - MODULE_NAME=process_file.app
       - KAFKA_BOOTSTRAP_SERVERS=kafka:9093
+      - LOG_LEVEL=DEBUG
       - C_FORCE_ROOT=true # needed when using pickle serializer in celery + running as root - remove when we dont run as root
     command:
       'sh -c "cd tasks && poetry run celery -A $$MODULE_NAME worker --loglevel=info"'

server/README.md

@@ -80,3 +80,9 @@ cd ui/
 yarn codegen
 ```
 now edit `src/adala.tsx` manually given the autogenerated changes in `src/_api/`
+
+## change the log level
+
+If not set, the logging level will default to "INFO".
+
+When doing native dev, you can set the `LOG_LEVEL` env var to "INFO", "DEBUG", etc. For docker, change the value of the `LOG_LEVEL` env var in `docker-compose.yml` (for both `app` and `worker`).

server/app.py

@@ -1,4 +1,3 @@
-import logging
 from enum import Enum
 from typing import Any, Dict, Generic, List, Optional, TypeVar
 import os
@@ -15,18 +14,13 @@
 from redis import Redis
 import time
 
+from server.handlers.result_handlers import ResultHandler
 from server.log_middleware import LogMiddleware
 from server.tasks.process_file import streaming_parent_task
-from server.utils import (
-    get_input_topic_name,
-    get_output_topic_name,
-    Settings,
-    delete_topic,
-)
-from server.handlers.result_handlers import ResultHandler
-
+from server.utils import (Settings, delete_topic, get_input_topic_name,
+                          get_output_topic_name, init_logger)
 
-logger = logging.getLogger(__name__)
+logger = init_logger(__name__)
 
 
 settings = Settings()

server/handlers/result_handlers.py

@@ -1,14 +1,13 @@
 from typing import Optional
-import logging
 import json
 from abc import abstractmethod
 from pydantic import BaseModel, Field, computed_field, ConfigDict, model_validator
 import csv
 
 from adala.utils.registry import BaseModelInRegistry
+from server.utils import init_logger
 
-
-logger = logging.getLogger(__name__)
+logger = init_logger(__name__)
 
 try:
     from label_studio_sdk import Client as LSEClient
@@ -149,6 +148,8 @@ def __call__(self, result_batch: list[LSEBatchItem]):
             if result.get('output') != result.get('output'):
                 result['output'] = None
 
+            logger.debug('Record in LSEHandler: %s', result)
+
             norm_result_batch.append(LSEBatchItem(**result))
 
         # omit failed tasks for now

server/tasks/process_file.py

@@ -1,25 +1,19 @@
 import asyncio
 import json
 import os
-import logging
 import time
 
 from adala.agents import Agent
 
 from aiokafka import AIOKafkaConsumer
 from aiokafka.errors import UnknownTopicOrPartitionError
 from celery import Celery
-from server.utils import (
-    get_input_topic_name,
-    get_output_topic_name,
-    ensure_topic,
-    delete_topic,
-    Settings,
-)
 from server.handlers.result_handlers import ResultHandler
+from server.utils import (Settings, delete_topic, ensure_topic,
+                          get_input_topic_name, get_output_topic_name,
+                          init_logger)
 
-
-logger = logging.getLogger(__name__)
+logger = init_logger(__name__)
 
 REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
 app = Celery(

server/utils.py

@@ -1,9 +1,13 @@
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import List, Union
+import logging
+import os
 from pathlib import Path
 from kafka.admin import KafkaAdminClient, NewTopic
 from kafka.errors import TopicAlreadyExistsError
 
+LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO').upper()
+
 
 class Settings(BaseSettings):
     """
@@ -71,3 +75,19 @@ def delete_topic(topic_name: str):
     )
 
     admin_client.delete_topics(topics=[topic_name])
+
+
+def init_logger(name, level=LOG_LEVEL):
+    """Set up a logger that respects the LOG_LEVEL env var
+
+    Args:
+        name (str): the name of the logger, typically the __name__ of the module
+        level (Union[str,int]): the logging level to use
+            (either a string like "INFO" or an int coming from the logging
+            module, like logging.INFO)
+    Returns:
+        logging.Logger
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    return logger