pre-processing-v0.0.1-beta-SNAPSHOT-9

.github/workflows/deploy-fastapi.yml

@@ -31,6 +31,9 @@ jobs:
           echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env.prod
           echo "MODE=prd" >> .env.prod
           echo "APP_NAME=pre-processing" >> .env.prod
+          echo "GRAFANA_CLOUD_PROMETHEUS_URL=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_URL }}" >> .env.prod
+          echo "GRAFANA_CLOUD_PROMETHEUS_USER=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_USER }}" >> .env.prod
+          echo "GRAFANA_CLOUD_API_KEY=${{ secrets.GRAFANA_CLOUD_API_KEY }}" >> .env.prod
 
       - name: Set repo lowercase
         run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
@@ -64,6 +67,16 @@ jobs:
           target: "~/app"
           overwrite: true
 
+      - name: Copy agent-config to FastAPI EC2
+        uses: appleboy/[email protected]
+        with:
+          host: ${{ secrets.FASTAPI_SERVER_HOST }}
+          username: ubuntu
+          key: ${{ secrets.SERVER_SSH_KEY }}
+          source: "docker/production-fastapi/agent-config.yml"
+          target: "~/app"
+          overwrite: true
+
       - name: Deploy on EC2
         uses: appleboy/[email protected]
         with:

apps/pre-processing-service/app/core/config.py

@@ -87,6 +87,10 @@ class BaseSettingsConfig(BaseSettings):
     loki_password: str
     loki_port: int = 3100
 
+    grafana_cloud_prometheus_url: Optional[str] = None
+    grafana_cloud_prometheus_user: Optional[str] = None
+    grafana_cloud_api_key: Optional[str] = None
+
     # 테스트/추가용 필드
     openai_api_key: Optional[str] = None  # << 이 부분 추가
 

apps/pre-processing-service/app/core/logging_config.py

@@ -14,17 +14,22 @@
 def setup_file_logging():
     """
     PromTail을 통해 Loki로 전송하기 위한 파일 로깅 설정
+    환경변수 MODE에 따라 개발/운영 환경 분리
     """
     # 기존 loguru 핸들러 제거 (기본 콘솔 출력 제거)
     logger.remove()
 
-    log_dir = "../../docker/local/logs/develop"
+    mode = os.getenv("MODE", "dev").lower()
 
-    # 로그 디렉토리가 없으면 생성
-
-    # 로그 파일 경로 설정
-    log_file_path = log_dir + "/pre-processing-app.log"
-    error_log_file_path = log_dir + "/pre-processing-app-error.log"
+    if mode == "prd":
+        log_dir = "/logs/production"
+        log_file_path = f"{log_dir}/app.log"
+        error_log_file_path = f"{log_dir}/error.log"
+    else:
+        # 개발환경: 기존 로컬 경로
+        log_dir = "../../docker/local/logs/develop"
+        log_file_path = f"{log_dir}/pre-processing-app.log"
+        error_log_file_path = f"{log_dir}/pre-processing-app-error.log"
 
     # trace_id를 포함한 간단한 포맷 문자열 사용
     def add_trace_id_filter(record):

apps/pre-processing-service/app/main.py

@@ -4,6 +4,7 @@
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from fastapi.exceptions import RequestValidationError
 from app.middleware.ServiceLoggerMiddleware import ServiceLoggerMiddleware
+from prometheus_fastapi_instrumentator import Instrumentator
 
 # 파일 로깅 설정 초기화
 from app.core.logging_config import setup_file_logging
@@ -19,6 +20,11 @@
 # --- FastAPI 애플리케이션 인스턴스 생성 ---
 app = FastAPI(title="pre-processing-service", description="", version="1.0.0")
 
+# --- Prometheus 메트릭스 설정 ---
+instrumentator = Instrumentator()
+instrumentator.instrument(app)
+instrumentator.expose(app, endpoint="/actuator/prometheus")
+
 # --- 예외 핸들러 등록 ---
 # 등록 순서가 중요합니다: 구체적인 예외부터 등록하고 가장 일반적인 예외(Exception)를 마지막에 등록합니다.
 app.add_exception_handler(CustomException, custom_exception_handler)

apps/pre-processing-service/pyproject.toml

@@ -36,6 +36,8 @@ dbutils=">=3.1.2,<4.0.0"
 onnxruntime = "^1.22.1"
 openai = "^1.107.3"
 aiohttp = "^3.12.15"
+prometheus-client = "^0.23.1"
+prometheus-fastapi-instrumentator = "^7.1.0"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/FastApiTaskRunner.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/FastApiTaskRunner.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner;
+package site.icebang.domain.workflow.runner.fastapi;
 
 import org.springframework.http.HttpMethod;
 import org.springframework.stereotype.Component;
@@ -10,6 +10,7 @@
 
 import site.icebang.domain.execution.model.TaskRun;
 import site.icebang.domain.workflow.model.Task;
+import site.icebang.domain.workflow.runner.TaskRunner;
 import site.icebang.external.fastapi.adapter.FastApiAdapter;
 
 @Component("fastapiTaskRunner")

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogPublishBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogPublishBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 import java.util.Optional;

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogRagBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogRagBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 import java.util.Optional;

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/KeywordSearchBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/KeywordSearchBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductCrawlBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductCrawlBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 import java.util.Optional;

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductMatchBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductMatchBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 import java.util.Optional;

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSearchBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSearchBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSimilarityBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSimilarityBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 import java.util.Optional;

apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/TaskBodyBuilder.java → apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/TaskBodyBuilder.java

@@ -1,4 +1,4 @@
-package site.icebang.domain.workflow.runner.body;
+package site.icebang.domain.workflow.runner.fastapi.body;
 
 import java.util.Map;
 

apps/user-service/src/main/java/site/icebang/domain/workflow/service/WorkflowExecutionService.java

@@ -27,7 +27,7 @@
 import site.icebang.domain.workflow.model.Job;
 import site.icebang.domain.workflow.model.Task;
 import site.icebang.domain.workflow.runner.TaskRunner;
-import site.icebang.domain.workflow.runner.body.TaskBodyBuilder;
+import site.icebang.domain.workflow.runner.fastapi.body.TaskBodyBuilder;
 
 @Slf4j
 @Service

docker/production-fastapi/agent-config.yml

@@ -0,0 +1,32 @@
+server:
+  log_level: info
+
+prometheus:
+  wal_directory: /tmp/grafana-agent-wal
+  global:
+    scrape_interval: 15s
+    external_labels:
+      cluster: production
+      service: pre-processing-service
+
+  configs:
+    - name: pre-processing-service-metrics
+      remote_write:
+        - url: ${GRAFANA_CLOUD_PROMETHEUS_URL}
+          basic_auth:
+            username: ${GRAFANA_CLOUD_PROMETHEUS_USER}
+            password: ${GRAFANA_CLOUD_API_KEY}
+
+      scrape_configs:
+        - job_name: 'pre-processing-service'
+          static_configs:
+            - targets: ['pre-processing-service:8000']
+          metrics_path: '/actuator/prometheus'
+          scrape_interval: 15s
+          params:
+            format: ['prometheus']
+
+        - job_name: 'node-exporter'
+          static_configs:
+            - targets: ['node-exporter:9100']
+          scrape_interval: 15s

docker/production-fastapi/docker-compose.yml

@@ -14,6 +14,8 @@ services:
       - promtail
     env_file:
       - .env.prod
+    networks:
+      - app-network
 
   promtail:
     image: grafana/promtail:2.9.0
@@ -31,7 +33,47 @@ services:
         hard: 65535
     env_file:
       - .env.prod
+    networks:
+      - app-network
+
+  grafana-agent:
+    image: grafana/agent:latest
+    container_name: grafana-agent
+    restart: unless-stopped
+    volumes:
+      - ./agent-config.yml:/etc/agent/agent.yml:ro
+    networks:
+      - app-network
+    env_file:
+      - .env.prod
+    command:
+      - --config.file=/etc/agent/agent.yml
+      - --config.expand-env=true
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: node-exporter
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:9100:9100"  # localhost에서만 접근
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
+    networks:
+      - app-network
+
+
 
 volumes:
   logs_volume:
     driver: local
+
+networks:
+  app-network:
+    driver: bridge