practice

.github/workflows/test.yml

@@ -25,17 +25,17 @@ jobs:
 
     - name: Lint with flake8
       run: |
-        flake8 day5/演習3 --count --select=E9,F63,F7,F82 --show-source --statistics
-        flake8 day5/演習3 --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
+        flake8 day5/practice3 --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 day5/practice3 --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
 
     - name: Format check with black
       run: |
-        black --check day5/演習3
+        black --check day5/practice3
 
     - name: Run data tests
       run: |
-        pytest day5/演習3/tests/test_data.py -v
+        pytest day5/practice3/tests/test_data.py -v
 
     - name: Run model tests
       run: |
-        pytest day5/演習3/tests/test_model.py -v
+        pytest day5/practice3/tests/test_model.py -v

day5/演習1/main.py → day5/practice1/main.py

@@ -10,6 +10,7 @@
 from sklearn.metrics import accuracy_score
 from sklearn.preprocessing import LabelEncoder
 from mlflow.models.signature import infer_signature
+import time
 
 
 # データ準備
@@ -76,6 +77,14 @@ def log_model(model, accuracy, params):
         print(f"モデルのログ記録値 \naccuracy: {accuracy}\nparams: {params}")
 
 
+def evaluate_inference_time_and_accuracy(model, X_test, y_test):
+    start_time = time.time()
+    predictions = model.predict(X_test)
+    inference_time = time.time() - start_time
+    accuracy = accuracy_score(y_test, predictions)
+    return inference_time, accuracy
+
+
 # メイン処理
 if __name__ == "__main__":
     # ランダム要素の設定
@@ -121,3 +130,21 @@ def log_model(model, accuracy, params):
     with open(model_path, "wb") as f:
         pickle.dump(model, f)
     print(f"モデルを {model_path} に保存しました")
+
+    # モデルの推論精度と時間を評価
+    inference_time, current_accuracy = evaluate_inference_time_and_accuracy(model, X_test, y_test)
+    mlflow.log_metric("inference_time", inference_time)
+    mlflow.log_metric("current_accuracy", current_accuracy)
+
+    # 過去バージョンのモデルと比較
+    past_model_path = os.path.join(model_dir, "titanic_model.pkl")
+    if os.path.exists(past_model_path):
+        with open(past_model_path, "rb") as f:
+            past_model = pickle.load(f)
+        past_inference_time, past_accuracy = evaluate_inference_time_and_accuracy(past_model, X_test, y_test)
+        mlflow.log_metric("past_inference_time", past_inference_time)
+        mlflow.log_metric("past_accuracy", past_accuracy)
+        print(f"過去モデルの精度: {past_accuracy}, 推論時間: {past_inference_time}")
+        print(f"現在モデルの精度: {current_accuracy}, 推論時間: {inference_time}")
+    else:
+        print("過去モデルが見つかりませんでした。")

day5/演習3/tests/test_model.py → day5/practice3/tests/test_model.py

@@ -171,3 +171,35 @@ def test_model_reproducibility(sample_data, preprocessor):
     assert np.array_equal(
         predictions1, predictions2
     ), "モデルの予測結果に再現性がありません"
+
+
+def test_model_performance_comparison(train_model):
+    """現在のモデルと過去のモデルの性能を比較"""
+    model, X_test, y_test = train_model
+
+    # 現在のモデルの精度と推論時間を計算
+    current_y_pred = model.predict(X_test)
+    current_accuracy = accuracy_score(y_test, current_y_pred)
+    start_time = time.time()
+    model.predict(X_test)
+    current_inference_time = time.time() - start_time
+
+    # 過去のモデルをロードして比較
+    if os.path.exists(MODEL_PATH):
+        with open(MODEL_PATH, "rb") as f:
+            past_model = pickle.load(f)
+        past_y_pred = past_model.predict(X_test)
+        past_accuracy = accuracy_score(y_test, past_y_pred)
+        start_time = time.time()
+        past_model.predict(X_test)
+        past_inference_time = time.time() - start_time
+
+        # 精度と推論時間を比較
+        assert (
+            current_accuracy >= past_accuracy
+        ), f"現在のモデルの精度が過去のモデルより低いです: {current_accuracy} < {past_accuracy}"
+        assert (
+            current_inference_time <= past_inference_time
+        ), f"現在のモデルの推論時間が過去のモデルより長いです: {current_inference_time} > {past_inference_time}"
+    else:
+        pytest.skip("過去のモデルが存在しないためスキップします")