Merge pull request #51 from HIGHFIVE-SW/develop

Develop

Author: urusekai

app.py

@@ -33,6 +33,9 @@
 from ocr import ocr_bp
 app.register_blueprint(ocr_bp)
 
+from crawler import crawler_bp
+app.register_blueprint(crawler_bp)
+
 swagger=Swagger(app)
 
 # logger.debug('DEBUG logging test.')

crawler/__init__.py

@@ -2,22 +2,88 @@
 from crawler.main_crawler import run_crawlers
 from server.logger import logger
 
-crawler_bp = Blueprint('crawler', __name__, url_prefix='/crawler') 
+import threading
 
-@crawler_bp.route('/', methods=['GET'])
-def crawler():
-    # GET 파라미터에서 targets 가져오기
-    targets_param = request.args.get('targets')  # 예: 'bbc,wevity'
+crawler_bp = Blueprint('crawler', __name__, url_prefix='/crawler')
 
-    # 파라미터가 있다면 리스트로 분할
-    if targets_param:
-        targets = [t.strip() for t in targets_param.split(',')]
-    else:
-        targets = None  # 전체 실행
+# 상태 관리 변수
+status = {
+    "state": "idle",     # idle | running | done | error
+    "error": None,
+    "targets": None
+}
+lock = threading.Lock()
+
+def background_crawl(targets):
+    global status
 
     try:
+        with lock:
+            status["state"] = "running"
+            status["error"] = None
+            status["targets"] = targets or "all"
+
         run_crawlers(targets)
-        return jsonify({"message": "크롤러 실행 완료", "targets": targets or "all"}), 200
+
+        with lock:
+            status["state"] = "done"
     except Exception as e:
         logger.error(f"크롤러 실행 중 오류: {e}")
-        return jsonify({"error": str(e)}), 500
+        with lock:
+            status["state"] = "error"
+            status["error"] = str(e)
+    finally:
+        pass
+
+@crawler_bp.route('/run', methods=['GET'])
+def start_crawler():
+    """크롤러를 시작하는 API 엔드포인트
+    
+    Query Parameters:
+        targets (str, optional): 쉼표로 구분된 크롤링 대상 목록
+                                예: "bbc,wevity"
+                                미지정시 모든 대상을 크롤링
+    
+    Returns:
+        JSON: 
+            - 성공시 (202):
+                {
+                    "message": "크롤러 실행 시작됨",
+                    "targets": "<target_list>" or "all"
+                }
+            - 실행 중일 때 (429):
+                {
+                    "message": "이미 크롤러가 실행 중입니다."
+                }
+    """
+    targets_param = request.args.get('targets')
+    targets = [t.strip() for t in targets_param.split(',')] if targets_param else None
+
+    with lock:
+        if status["state"] == "running":
+            return jsonify({"message": "이미 크롤러가 실행 중입니다."}), 429
+
+        thread = threading.Thread(target=background_crawl, args=(targets,))
+        thread.start()
+
+    return jsonify({"message": "크롤러 실행 시작됨", "targets": targets or "all"}), 202
+
+
+@crawler_bp.route('/status', methods=['GET'])
+def get_crawler_status():
+    """현재 크롤러의 상태를 조회하는 API 엔드포인트
+    
+    Returns:
+        JSON (200):
+            {
+                "state": "idle" | "running" | "done" | "error",
+                "error": "오류 메시지" or null,
+                "targets": ["대상1", "대상2"] or "all" or null
+            }
+    """
+    with lock:
+        return jsonify({
+            "state": status["state"],
+            "error": status["error"],
+            "targets": status["targets"]
+        }), 200

crawler/bbc_crawler.py

@@ -9,7 +9,9 @@
 COLLECTIONS = {
     'natural-wonders' : '9f0b9075-b620-4859-abdc-ed042dd9ee66',
     'weather-science' : '696fca43-ec53-418d-a42c-067cb0449ba9',
-    'climate-solutions' : '5fa7bbe8-5ea3-4bc6-ac7e-546d0dc4a16b',
+    'climate-solutions' : '5fa7bbe8-5ea3-4bc6-ac7e-546d0dc4a16b',   
+    'world' : '07cedf01-f642-4b92-821f-d7b324b8ba73',
+    'innovation' : '3da03ce0-ee41-4427-a5d9-1294491e0448'
 }
 HEADERS = {
     'User-Agent': 'Mozilla/5.0'
@@ -76,7 +78,6 @@ def get_articles(page, collection_id, end_time):
         url = "https://www.bbc.com" + data['path']
         content = get_content(url)
         title, content, keyword = summarize_and_categorize_issue(data['title'], content) 
-        print(content)
         image = data['indexImage']['model']['blocks']['src'] or None
 
         articles.append(

server/logger.py

@@ -181,6 +181,10 @@ def filter(self, record):
 
 logger = logging.getLogger(__name__)
 
+# 로깅 제거
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
 
 if __name__ == '__main__':
     # 모듈 사용 테스트