diff --git a/.github/workflows/deploy-fastapi.yml b/.github/workflows/deploy-fastapi.yml index a87c064d..7d68d7bd 100644 --- a/.github/workflows/deploy-fastapi.yml +++ b/.github/workflows/deploy-fastapi.yml @@ -31,6 +31,9 @@ jobs: echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> .env.prod echo "MODE=prd" >> .env.prod echo "APP_NAME=pre-processing" >> .env.prod + echo "GRAFANA_CLOUD_PROMETHEUS_URL=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_URL }}" >> .env.prod + echo "GRAFANA_CLOUD_PROMETHEUS_USER=${{ secrets.GRAFANA_CLOUD_PROMETHEUS_USER }}" >> .env.prod + echo "GRAFANA_CLOUD_API_KEY=${{ secrets.GRAFANA_CLOUD_API_KEY }}" >> .env.prod - name: Set repo lowercase run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV @@ -64,6 +67,16 @@ jobs: target: "~/app" overwrite: true + - name: Copy agent-config to FastAPI EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.FASTAPI_SERVER_HOST }} + username: ubuntu + key: ${{ secrets.SERVER_SSH_KEY }} + source: "docker/production-fastapi/agent-config.yml" + target: "~/app" + overwrite: true + - name: Deploy on EC2 uses: appleboy/ssh-action@v1.0.3 with: diff --git a/apps/pre-processing-service/app/core/config.py b/apps/pre-processing-service/app/core/config.py index c6f31f49..9580934b 100644 --- a/apps/pre-processing-service/app/core/config.py +++ b/apps/pre-processing-service/app/core/config.py @@ -87,6 +87,10 @@ class BaseSettingsConfig(BaseSettings): loki_password: str loki_port: int = 3100 + grafana_cloud_prometheus_url: Optional[str] = None + grafana_cloud_prometheus_user: Optional[str] = None + grafana_cloud_api_key: Optional[str] = None + # 테스트/추가용 필드 openai_api_key: Optional[str] = None # << 이 부분 추가 diff --git a/apps/pre-processing-service/app/core/logging_config.py b/apps/pre-processing-service/app/core/logging_config.py index bffca718..a511aee2 100644 --- a/apps/pre-processing-service/app/core/logging_config.py +++ b/apps/pre-processing-service/app/core/logging_config.py @@ -14,17 +14,22 @@ def setup_file_logging(): """ PromTail을 통해 Loki로 전송하기 위한 파일 로깅 설정 + 환경변수 MODE에 따라 개발/운영 환경 분리 """ # 기존 loguru 핸들러 제거 (기본 콘솔 출력 제거) logger.remove() - log_dir = "../../docker/local/logs/develop" + mode = os.getenv("MODE", "dev").lower() - # 로그 디렉토리가 없으면 생성 - - # 로그 파일 경로 설정 - log_file_path = log_dir + "/pre-processing-app.log" - error_log_file_path = log_dir + "/pre-processing-app-error.log" + if mode == "prd": + log_dir = "/logs/production" + log_file_path = f"{log_dir}/app.log" + error_log_file_path = f"{log_dir}/error.log" + else: + # 개발환경: 기존 로컬 경로 + log_dir = "../../docker/local/logs/develop" + log_file_path = f"{log_dir}/pre-processing-app.log" + error_log_file_path = f"{log_dir}/pre-processing-app-error.log" # trace_id를 포함한 간단한 포맷 문자열 사용 def add_trace_id_filter(record): diff --git a/apps/pre-processing-service/app/main.py b/apps/pre-processing-service/app/main.py index 4bbf3ff1..1ea95694 100644 --- a/apps/pre-processing-service/app/main.py +++ b/apps/pre-processing-service/app/main.py @@ -4,6 +4,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException from fastapi.exceptions import RequestValidationError from app.middleware.ServiceLoggerMiddleware import ServiceLoggerMiddleware +from prometheus_fastapi_instrumentator import Instrumentator # 파일 로깅 설정 초기화 from app.core.logging_config import setup_file_logging @@ -19,6 +20,11 @@ # --- FastAPI 애플리케이션 인스턴스 생성 --- app = FastAPI(title="pre-processing-service", description="", version="1.0.0") +# --- Prometheus 메트릭스 설정 --- +instrumentator = Instrumentator() +instrumentator.instrument(app) +instrumentator.expose(app, endpoint="/actuator/prometheus") + # --- 예외 핸들러 등록 --- # 등록 순서가 중요합니다: 구체적인 예외부터 등록하고 가장 일반적인 예외(Exception)를 마지막에 등록합니다. app.add_exception_handler(CustomException, custom_exception_handler) diff --git a/apps/pre-processing-service/poetry.lock b/apps/pre-processing-service/poetry.lock index 1c39c240..ca5c20ab 100644 --- a/apps/pre-processing-service/poetry.lock +++ b/apps/pre-processing-service/poetry.lock @@ -1803,6 +1803,37 @@ files = [ {file = "poetry_core-2.2.0.tar.gz", hash = "sha256:b4033b71b99717a942030e074fec7e3082e5fde7a8ed10f02cd2413bdf940b1f"}, ] +[[package]] +name = "prometheus-client" +version = "0.23.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99"}, + {file = "prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "7.1.0" +description = "Instrument your FastAPI app with Prometheus metrics" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, + {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, +] + +[package.dependencies] +prometheus-client = ">=0.8.0,<1.0.0" +starlette = ">=0.30.0,<1.0.0" + [[package]] name = "propcache" version = "0.3.2" @@ -3398,4 +3429,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "6e10697924e89b5c0f7c3f6ecd79fb64ac412ac2240f75565d6ea5feb3a89a20" +content-hash = "42274fd00aedabf70dc419acd06e2f25b5c69b58b7bf76eef2ea7a9df6470b2c" diff --git a/apps/pre-processing-service/pyproject.toml b/apps/pre-processing-service/pyproject.toml index 672bf645..84a957b9 100644 --- a/apps/pre-processing-service/pyproject.toml +++ b/apps/pre-processing-service/pyproject.toml @@ -36,6 +36,8 @@ dbutils=">=3.1.2,<4.0.0" onnxruntime = "^1.22.1" openai = "^1.107.3" aiohttp = "^3.12.15" +prometheus-client = "^0.23.1" +prometheus-fastapi-instrumentator = "^7.1.0" [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/FastApiTaskRunner.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/FastApiTaskRunner.java similarity index 91% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/FastApiTaskRunner.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/FastApiTaskRunner.java index 5a36afa3..c7a62c97 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/FastApiTaskRunner.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/FastApiTaskRunner.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner; +package site.icebang.domain.workflow.runner.fastapi; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Component; @@ -10,6 +10,7 @@ import site.icebang.domain.execution.model.TaskRun; import site.icebang.domain.workflow.model.Task; +import site.icebang.domain.workflow.runner.TaskRunner; import site.icebang.external.fastapi.adapter.FastApiAdapter; @Component("fastapiTaskRunner") diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogPublishBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogPublishBodyBuilder.java similarity index 97% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogPublishBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogPublishBodyBuilder.java index 07f0480b..94613f64 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogPublishBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogPublishBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; import java.util.Optional; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogRagBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogRagBodyBuilder.java similarity index 96% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogRagBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogRagBodyBuilder.java index 49ff52e8..ad22a58d 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/BlogRagBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/BlogRagBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; import java.util.Optional; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/KeywordSearchBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/KeywordSearchBodyBuilder.java similarity index 93% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/KeywordSearchBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/KeywordSearchBodyBuilder.java index f1bd5509..17add786 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/KeywordSearchBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/KeywordSearchBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductCrawlBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductCrawlBodyBuilder.java similarity index 95% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductCrawlBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductCrawlBodyBuilder.java index 5522b7fb..138e95d0 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductCrawlBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductCrawlBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; import java.util.Optional; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductMatchBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductMatchBodyBuilder.java similarity index 96% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductMatchBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductMatchBodyBuilder.java index 4d92bde3..65e693f3 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductMatchBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductMatchBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; import java.util.Optional; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSearchBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSearchBodyBuilder.java similarity index 94% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSearchBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSearchBodyBuilder.java index e7cc9f89..d594e4e2 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSearchBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSearchBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSimilarityBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSimilarityBodyBuilder.java similarity index 96% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSimilarityBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSimilarityBodyBuilder.java index d8964f2c..45f19ad8 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/ProductSimilarityBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/ProductSimilarityBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; import java.util.Optional; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/TaskBodyBuilder.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/TaskBodyBuilder.java similarity index 92% rename from apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/TaskBodyBuilder.java rename to apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/TaskBodyBuilder.java index 0d807d86..04dacef4 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/body/TaskBodyBuilder.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/runner/fastapi/body/TaskBodyBuilder.java @@ -1,4 +1,4 @@ -package site.icebang.domain.workflow.runner.body; +package site.icebang.domain.workflow.runner.fastapi.body; import java.util.Map; diff --git a/apps/user-service/src/main/java/site/icebang/domain/workflow/service/WorkflowExecutionService.java b/apps/user-service/src/main/java/site/icebang/domain/workflow/service/WorkflowExecutionService.java index d142b630..60da5863 100644 --- a/apps/user-service/src/main/java/site/icebang/domain/workflow/service/WorkflowExecutionService.java +++ b/apps/user-service/src/main/java/site/icebang/domain/workflow/service/WorkflowExecutionService.java @@ -27,7 +27,7 @@ import site.icebang.domain.workflow.model.Job; import site.icebang.domain.workflow.model.Task; import site.icebang.domain.workflow.runner.TaskRunner; -import site.icebang.domain.workflow.runner.body.TaskBodyBuilder; +import site.icebang.domain.workflow.runner.fastapi.body.TaskBodyBuilder; @Slf4j @Service diff --git a/docker/production-fastapi/agent-config.yml b/docker/production-fastapi/agent-config.yml new file mode 100644 index 00000000..7119a4e3 --- /dev/null +++ b/docker/production-fastapi/agent-config.yml @@ -0,0 +1,32 @@ +server: + log_level: info + +prometheus: + wal_directory: /tmp/grafana-agent-wal + global: + scrape_interval: 15s + external_labels: + cluster: production + service: pre-processing-service + + configs: + - name: pre-processing-service-metrics + remote_write: + - url: ${GRAFANA_CLOUD_PROMETHEUS_URL} + basic_auth: + username: ${GRAFANA_CLOUD_PROMETHEUS_USER} + password: ${GRAFANA_CLOUD_API_KEY} + + scrape_configs: + - job_name: 'pre-processing-service' + static_configs: + - targets: ['pre-processing-service:8000'] + metrics_path: '/actuator/prometheus' + scrape_interval: 15s + params: + format: ['prometheus'] + + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + scrape_interval: 15s \ No newline at end of file diff --git a/docker/production-fastapi/docker-compose.yml b/docker/production-fastapi/docker-compose.yml index 8dd5dbb8..c22a3833 100644 --- a/docker/production-fastapi/docker-compose.yml +++ b/docker/production-fastapi/docker-compose.yml @@ -14,6 +14,8 @@ services: - promtail env_file: - .env.prod + networks: + - app-network promtail: image: grafana/promtail:2.9.0 @@ -31,7 +33,47 @@ services: hard: 65535 env_file: - .env.prod + networks: + - app-network + + grafana-agent: + image: grafana/agent:latest + container_name: grafana-agent + restart: unless-stopped + volumes: + - ./agent-config.yml:/etc/agent/agent.yml:ro + networks: + - app-network + env_file: + - .env.prod + command: + - --config.file=/etc/agent/agent.yml + - --config.expand-env=true + + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + restart: unless-stopped + ports: + - "127.0.0.1:9100:9100" # localhost에서만 접근 + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + networks: + - app-network + + volumes: logs_volume: driver: local + +networks: + app-network: + driver: bridge