배포 v1.12.1

배포 v1.12.1

Author: skqorrla

.github/workflows/Deploy-ML-EC2.yml

@@ -41,13 +41,17 @@ jobs:
         SIX_HOUR_MODEL_PATH: ${{ secrets.SIX_HOUR_MODEL_PATH }}
         TWELVE_HOUR_MODEL_PATH: ${{ secrets.TWELVE_HOUR_MODEL_PATH }}
 
-        PRED_PATH: ${{ secrets.PRED_PATH }}
+        # PRED_PATH: ${{ secrets.PRED_PATH }}
+        S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
+        S3_DIRECTORY_PATH: ${{ secrets.S3_DIRECTORY_PATH }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       with:
         host: ${{ secrets.EC2_HOST_ML }}
         username: ${{ secrets.EC2_USER }}
         key: ${{ secrets.EC2_SSH_KEY }}
         port: 22
-        envs: OPENAI_API_KEY,VECTORDB_PATH,DB_HOST,DB_PORT,DB_USER,DB_PASSWORD,DB_DATABASE,CLICK_LOG,DEEPFM_TRAIN_MODEL_PATH,DEEPFM_TRAIN_ENCODERS_PATH,DEEPFM_TRAIN_KEY2INDEX_PATH,SEOUL_API_KEY,PLACE_PATH,ONE_HOUR_MODEL_PATH,TWO_HOUR_MODEL_PATH,THREE_HOUR_MODEL_PATH,SIX_HOUR_MODEL_PATH,TWELVE_HOUR_MODEL_PATH,PRED_PATH
+        envs: OPENAI_API_KEY,VECTORDB_PATH,DB_HOST,DB_PORT,DB_USER,DB_PASSWORD,DB_DATABASE,CLICK_LOG,DEEPFM_TRAIN_MODEL_PATH,DEEPFM_TRAIN_ENCODERS_PATH,DEEPFM_TRAIN_KEY2INDEX_PATH,SEOUL_API_KEY,PLACE_PATH,ONE_HOUR_MODEL_PATH,TWO_HOUR_MODEL_PATH,THREE_HOUR_MODEL_PATH,SIX_HOUR_MODEL_PATH,TWELVE_HOUR_MODEL_PATH,PRED_PATH,S3_BUCKET_NAME,AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY
         script: |
           # 프로젝트 디렉터리로 이동
           cd ~/MLOps
@@ -87,6 +91,11 @@ jobs:
           echo "SEOUL_API_KEY=$SEOUL_API_KEY" >> .env
           echo "PLACE_PATH=$PLACE_PATH" >> .env
           echo "PRED_PATH=$PRED_PATH" >> .env
+
+          echo "S3_BUCKET_NAME=$S3_BUCKET_NAME" >> .env
+          echo "S3_DIRECTORY_PATH=$S3_DIRECTORY_PATH" >> .env
+          echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env
+          echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env
           
           # Docker 컨테이너 재시작 (docker-compose가 있다면)
           if [ -f docker-compose.yml ]; then

.github/workflows/dvc-sync.yml

@@ -6,11 +6,6 @@
 
 router = APIRouter(prefix="/api", tags=["crowdness"])
 
-# .env에서 PRED_PATH를 찾지 못할 경우를 대비한 기본 경로 설정
-PRED_PATH = os.getenv("PRED_PATH")
-if not PRED_PATH:
-    print("PRED_PATH is not set")
-
 @router.get("/crowd", response_model=CrowdResponse)
 def get_crowd_prediction(hour: int, area: str = "all"):
     """
@@ -19,18 +14,22 @@ def get_crowd_prediction(hour: int, area: str = "all"):
     - **area**: 조회할 지역 (all, 서울, 경기, 인천, 강원, 충청, 전라, 경상, 제주 중 하나)
     """
     try:
-        # 1. 대상 파일명 생성
+        # 1. 대상 S3 경로 생성
         target_timestamp = (datetime.now()).strftime("%Y%m%d%H")
         filename = f"congestion_predictions_{target_timestamp}_{hour}.csv"
-        file_path = os.path.join(PRED_PATH, filename)
-        
-        print(f"Attempting to read: {file_path}")
 
-        # 2. 파일 존재 확인 및 데이터 로드
-        if not os.path.exists(file_path):
-            raise HTTPException(status_code=404, detail=f"Prediction file not found for T+{hour}h. It may not have been generated yet.")
+        s3_bucket_name = os.getenv("S3_BUCKET_NAME")
+        if not s3_bucket_name:
+            raise ValueError("S3_BUCKET_NAME environment variable is not set.")
+        
+        s3_directory = os.getenv("S3_DIRECTORY_PATH", "")
+        s3_key = os.path.join(s3_directory, filename).replace("\\", "/") # S3는 /를 사용
+        s3_path = f"s3://{s3_bucket_name}/{s3_key}"
+        
+        print(f"Attempting to read from S3: {s3_path}")
 
-        df = pd.read_csv(file_path)
+        # 2. S3에서 데이터 로드
+        df = pd.read_csv(s3_path)
 
         # 3. 'area' 파라미터에 따른 데이터 필터링
         if area != "all":
@@ -60,5 +59,7 @@ def get_crowd_prediction(hour: int, area: str = "all"):
             message=f"조회 성공!",
             crowdLevel=crowd_level
         )
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail=f"Prediction file not found: {s3_path}. It may not have been generated yet.")
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}")

MLOps/app/python_schedule/data/.gitignore

@@ -8,6 +8,7 @@
 from collections import defaultdict
 import schedule
 import time
+import boto3
 
 # --- 경로 설정 및 초기화 ---
 project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
@@ -34,9 +35,6 @@
 # 파일명 접미사 매핑
 MODEL_FILE_SUFFIX = {"1-hour": 1, "2-hour": 2, "3-hour": 3, "6-hour": 6, "12-hour": 12}
 
-
-PRED_PATH = os.getenv("PRED_PATH")
-
 # --- 전역 객체 및 모델 로드 ---
 preprocessor = CongestionDataPreprocessor()
 models = {}
@@ -112,14 +110,6 @@ def predict_and_save_all_locations():
         print("No predictions were made.")
         return
 
-    # PRED_PATH가 설정되지 않았을 경우에 대한 예외 처리
-    if not PRED_PATH:
-        print("❌ PRED_PATH environment variable is not set. Cannot save files.")
-        return
-
-    # 저장 경로가 존재하는지 확인하고, 없으면 생성합니다.
-    os.makedirs(PRED_PATH, exist_ok=True)
-
     # 타임스탬프 (현재 시간 + 1시간)
     timestamp = (datetime.now() + timedelta(hours=1)).strftime("%Y%m%d%H")
 
@@ -139,10 +129,25 @@ def predict_and_save_all_locations():
 
         file_suffix = MODEL_FILE_SUFFIX.get(name)
         output_filename = f"congestion_predictions_{timestamp}_{file_suffix}.csv"
-        output_path = os.path.join(PRED_PATH, output_filename)
 
-        final_df.to_csv(output_path, index=False, encoding='utf-8-sig')
-        print(f"✅ Predictions for '{name}' saved to: {output_path}")
+        # --- S3에 바로 저장 ---
+        s3_bucket_name = os.getenv("S3_BUCKET_NAME")
+        s3_directory = os.getenv("S3_DIRECTORY_PATH", "") # S3 내 디렉토리 경로 (옵션)
+
+        if s3_bucket_name:
+            try:
+                # S3 키는 디렉토리 경로와 파일명을 조합하여 만듭니다.
+                s3_key = os.path.join(s3_directory, output_filename)
+                s3_path = f"s3://{s3_bucket_name}/{s3_key}"
+                
+                # EC2 인스턴스의 IAM 역할을 사용한 인증을 통해 S3에 직접 저장합니다.
+                final_df.to_csv(s3_path, index=False, encoding='utf-8-sig')
+                print(f"✅ Predictions for '{name}' successfully saved to: {s3_path}")
+
+            except Exception as e:
+                print(f"❌ S3 save failed: {e}")
+        else:
+            print("⚠️  S3_BUCKET_NAME environment variable not set. Skipping S3 save.")
 
 if __name__ == "__main__":
     print("Congestion prediction service started.")