Update for 2023-2024 Season (#290)

* Updated data for 2023-24 season

* Update xgboost ml train model file

* Updated main

* Update requirements.txt:

* Update NN model for ML

* Update NN model for UO

* Update get odds data

* Clean up NN runner

* Clan up get_data

* Clean up create games:

* Updated xgboost ml model

* Clean up xgboost uo model training

main.py

@@ -1,14 +1,14 @@
 import argparse
 from datetime import datetime, timedelta
-import time
-from colorama import Fore, Style
+
 import pandas as pd
 import tensorflow as tf
+from colorama import Fore, Style
+
+from src.DataProviders.SbrOddsProvider import SbrOddsProvider
 from src.Predict import NN_Runner, XGBoost_Runner
 from src.Utils.Dictionaries import team_index_current
 from src.Utils.tools import create_todays_games_from_odds, get_json_data, to_data_frame, get_todays_games_json, create_todays_games
-from src.DataProviders.SbrOddsProvider import SbrOddsProvider
-
 
 todays_games_url = 'https://data.nba.com/data/10s/v2015/json/mobile_teams/nba/2022/scores/00_todays_scores.json'
 data_url = 'https://stats.nba.com/stats/leaguedashteamstats?' \
@@ -26,7 +26,7 @@ def createTodaysGames(games, df, odds):
     todays_games_uo = []
     home_team_odds = []
     away_team_odds = []
-    # todo: get the days rest for current games
+
     home_team_days_rest = []
     away_team_days_rest = []
 
@@ -38,7 +38,7 @@ def createTodaysGames(games, df, odds):
         if odds is not None:
             game_odds = odds[home_team + ':' + away_team]
             todays_games_uo.append(game_odds['under_over_odds'])
-            
+
             home_team_odds.append(game_odds[home_team]['money_line_odds'])
             away_team_odds.append(game_odds[away_team]['money_line_odds'])
 
@@ -47,7 +47,7 @@ def createTodaysGames(games, df, odds):
 
             home_team_odds.append(input(home_team + ' odds: '))
             away_team_odds.append(input(away_team + ' odds: '))
-        
+
         # calculate days rest for both teams
         dateparse = lambda x: datetime.strptime(x, '%d/%m/%Y %H:%M')
         schedule_df = pd.read_csv('Data/nba-2023-UTC.csv', parse_dates=['Date'], date_parser=dateparse)
@@ -86,9 +86,9 @@ def main():
         if len(games) == 0:
             print("No games found.")
             return
-        if((games[0][0]+':'+games[0][1]) not in list(odds.keys())):
-            print(games[0][0]+':'+games[0][1])
-            print(Fore.RED, "--------------Games list not up to date for todays games!!! Scraping disabled until list is updated.--------------")
+        if (games[0][0] + ':' + games[0][1]) not in list(odds.keys()):
+            print(games[0][0] + ':' + games[0][1])
+            print(Fore.RED,"--------------Games list not up to date for todays games!!! Scraping disabled until list is updated.--------------")
             print(Style.RESET_ALL)
             odds = None
         else:

requirements.txt

@@ -1,8 +1,9 @@
 colorama==0.4.6
 pandas==2.1.1
-sbrscrape==0.0.8
+sbrscrape==0.0.9
 tensorflow==2.14.0
 tensorflow-metal==1.1.0
 xgboost==2.0.0
 tqdm==4.66.1
-flask==3.0.0
+flask==3.0.0
+scikit-learn==1.3.1

src/Predict/NN_Runner.py

@@ -1,9 +1,10 @@
 import copy
+
 import numpy as np
-import pandas as pd
 import tensorflow as tf
 from colorama import Fore, Style, init, deinit
-from tensorflow.keras.models import load_model
+from keras.models import load_model
+
 from src.Utils import Expected_Value
 from src.Utils import Kelly_Criterion as kc
 
@@ -70,8 +71,7 @@ def nn_runner(data, todays_games_uo, frame_ml, games, home_team_odds, away_team_
         if home_team_odds[count] and away_team_odds[count]:
             ev_home = float(Expected_Value.expected_value(ml_predictions_array[count][0][1], int(home_team_odds[count])))
             ev_away = float(Expected_Value.expected_value(ml_predictions_array[count][0][0], int(away_team_odds[count])))
-        expected_value_colors = {'home_color': Fore.GREEN if ev_home > 0 else Fore.RED,
-                        'away_color': Fore.GREEN if ev_away > 0 else Fore.RED}
+        expected_value_colors = {'home_color': Fore.GREEN if ev_home > 0 else Fore.RED, 'away_color': Fore.GREEN if ev_away > 0 else Fore.RED}
         bankroll_descriptor = ' Fraction of Bankroll: '
         bankroll_fraction_home = bankroll_descriptor + str(kc.calculate_kelly_criterion(home_team_odds[count], ml_predictions_array[count][0][1])) + '%'
         bankroll_fraction_away = bankroll_descriptor + str(kc.calculate_kelly_criterion(away_team_odds[count], ml_predictions_array[count][0][0])) + '%'

src/Process-Data/Create_Games.py

@@ -1,12 +1,14 @@
 import os
 import sqlite3
-import numpy as np
-import pandas as pd
 import sys
 from datetime import datetime
+
+import numpy as np
+import pandas as pd
 from tqdm import tqdm
+
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
-from Utils.Dictionaries import team_index_07, team_index_08, team_index_12, team_index_13, team_index_14, team_index_current
+from src.Utils.Dictionaries import team_index_07, team_index_08, team_index_12, team_index_13, team_index_14, team_index_current
 
 # season_array = ["2007-08", "2008-09", "2009-10", "2010-11", "2011-12", "2012-13", "2013-14", "2014-15", "2015-16",
 #                 "2016-17", "2017-18", "2018-19", "2019-20", "2020-21", "2021-22", "2022-23"]

src/Process-Data/Get_Data.py

@@ -1,14 +1,15 @@
 import os
 import random
 import sqlite3
-import time
 import sys
+import time
 from datetime import date, datetime, timedelta
 
 from tqdm import tqdm
 
+from src.Utils.tools import get_json_data, to_data_frame
+
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
-from Utils.tools import get_json_data, to_data_frame
 
 url = 'https://stats.nba.com/stats/' \
       'leaguedashteamstats?Conference=&' \
@@ -29,7 +30,8 @@
 #           "2017-18", "2018-19", "2019-20", "2020-2021", "2021-2022"]
 
 month = [10, 11, 12, 1, 2, 3, 4, 5, 6]
-days = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
+days = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+        31]
 
 begin_year_pointer = year[0]
 end_year_pointer = year[0]
@@ -45,7 +47,7 @@
         for day1 in tqdm(days):
             if month1 == 10 and day1 < 19:
                 continue
-            if month1 in [4,6,9,11] and day1 > 30:
+            if month1 in [4, 6, 9, 11] and day1 > 30:
                 continue
             if month1 == 2 and day1 > 28:
                 continue
@@ -61,8 +63,8 @@
 
             x = str(real_date).split('-')
             general_df.to_sql(f"teams_{season1}-{str(int(x[1]))}-{str(int(x[2]))}", con, if_exists="replace")
-            
+
             time.sleep(random.randint(1, 3))
     begin_year_pointer = year[count]
 
-con.close()
+con.close()

src/Process-Data/Get_Odds_Data.py

@@ -1,17 +1,18 @@
+import os
 import random
-import time
-import pandas as pd
 import sqlite3
-import os
 import sys
-
+import time
 from datetime import datetime, timedelta
-from tqdm import tqdm
+
+import pandas as pd
 from sbrscrape import Scoreboard
+from tqdm import tqdm
+
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
-from Utils.tools import get_date
+from src.Utils.tools import get_date
 
-year = [2022, 2023]
+year = ["2022", "2023"]
 season = ["2022-23"]
 
 month = [10, 11, 12, 1, 2, 3, 4, 5, 6]
@@ -21,7 +22,7 @@
 end_year_pointer = year[0]
 count = 0
 
-sportsbook='fanduel'
+sportsbook = 'fanduel'
 df_data = []
 
 con = sqlite3.connect("../../Data/odds.sqlite")
@@ -35,7 +36,7 @@
         for day1 in tqdm(days):
             if month1 == 10 and day1 < 19:
                 continue
-            if month1 in [4,6,9,11] and day1 > 30:
+            if month1 in [4, 6, 9, 11] and day1 > 30:
                 continue
             if month1 == 2 and day1 > 28:
                 continue
@@ -44,13 +45,14 @@
                     continue
                 if month1 > datetime.now().month:
                     continue
+            print(f"{end_year_pointer}-{month1:02}-{day1:02}")
             sb = Scoreboard(date=f"{end_year_pointer}-{month1:02}-{day1:02}")
             if not hasattr(sb, "games"):
                 continue
             for game in sb.games:
                 if game['home_team'] not in teams_last_played:
                     teams_last_played[game['home_team']] = get_date(f"{season1}-{month1:02}{day1:02}")
-                    home_games_rested = timedelta(days=7) # start of season, big number
+                    home_games_rested = timedelta(days=7)  # start of season, big number
                 else:
                     current_date = get_date(f"{season1}-{month1:02}{day1:02}")
                     home_games_rested = current_date - teams_last_played[game['home_team']]
@@ -59,12 +61,12 @@
 
                 if game['away_team'] not in teams_last_played:
                     teams_last_played[game['away_team']] = get_date(f"{season1}-{month1:02}{day1:02}")
-                    away_games_rested = timedelta(days=7) # start of season, big number
+                    away_games_rested = timedelta(days=7)  # start of season, big number
                 else:
                     current_date = get_date(f"{season1}-{month1:02}{day1:02}")
                     away_games_rested = current_date - teams_last_played[game['away_team']]
                     teams_last_played[game['away_team']] = current_date
-                
+
                 try:
                     df_data.append({
                         'Unnamed: 0': 0,
@@ -85,6 +87,6 @@
             time.sleep(random.randint(1, 3))
     begin_year_pointer = year[count]
 
-    df = pd.DataFrame(df_data,)
+    df = pd.DataFrame(df_data, )
     df.to_sql(f"odds_{season1}", con, if_exists="replace")
-con.close()
+con.close()

src/Train-Models/Train_Model.py → src/Train-Models/NN_Model_ML.py

@@ -1,9 +1,10 @@
 import sqlite3
 import time
+
 import numpy as np
 import pandas as pd
 import tensorflow as tf
-from tensorflow.keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint
+from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint
 
 current_time = str(time.time())
 
@@ -34,7 +35,6 @@
 model.add(tf.keras.layers.Dense(2, activation=tf.nn.softmax))
 
 model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
-model.fit(x_train, y_train, epochs=50, validation_split=0.1, batch_size=32,
-          callbacks=[tensorboard, earlyStopping, mcp_save])
+model.fit(x_train, y_train, epochs=50, validation_split=0.1, batch_size=32, callbacks=[tensorboard, earlyStopping, mcp_save])
 
 print('Done')

src/Train-Models/Train_Model_UO.py → src/Train-Models/NN_Model_UO.py

@@ -1,9 +1,10 @@
 import sqlite3
 import time
+
 import numpy as np
 import pandas as pd
 import tensorflow as tf
-from tensorflow.keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint
+from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint
 
 current_time = str(time.time())
 
@@ -35,7 +36,6 @@
 model.add(tf.keras.layers.Dense(3, activation=tf.nn.softmax))
 
 model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
-model.fit(x_train, y_train, epochs=50, validation_split=0.1, batch_size=32,
-          callbacks=[tensorboard, earlyStopping, mcp_save])
+model.fit(x_train, y_train, epochs=50, validation_split=0.1, batch_size=32, callbacks=[tensorboard, earlyStopping, mcp_save])
 
 print('Done')

src/Train-Models/XGBoost_Model_ML.py

@@ -1,10 +1,11 @@
 import sqlite3
+
+import numpy as np
 import pandas as pd
 import xgboost as xgb
 from sklearn.metrics import accuracy_score
 from sklearn.model_selection import train_test_split
 from tqdm import tqdm
-import numpy as np
 
 dataset = "dataset_2012-23"
 con = sqlite3.connect("../../Data/dataset.sqlite")
@@ -19,19 +20,19 @@
 
 data = data.astype(float)
 acc_results = []
-for x in tqdm(range(100)):
+for x in tqdm(range(300)):
     x_train, x_test, y_train, y_test = train_test_split(data, margin, test_size=.1)
 
     train = xgb.DMatrix(x_train, label=y_train)
     test = xgb.DMatrix(x_test, label=y_test)
 
     param = {
-        'max_depth': 2,
+        'max_depth': 3,
         'eta': 0.01,
         'objective': 'multi:softprob',
         'num_class': 2
     }
-    epochs = 500
+    epochs = 750
 
     model = xgb.train(param, train, epochs)
     predictions = model.predict(test)
@@ -40,9 +41,9 @@
     for z in predictions:
         y.append(np.argmax(z))
 
-    acc = round(accuracy_score(y_test, y)*100, 1)
+    acc = round(accuracy_score(y_test, y) * 100, 1)
     print(f"{acc}%")
     acc_results.append(acc)
     # only save results if they are the best so far
     if acc == max(acc_results):
-        model.save_model('../../Models/XGBoost_{}%_ML-2.json'.format(acc))
+        model.save_model('../../Models/XGBoost_{}%_ML-3.json'.format(acc))

src/Train-Models/XGBoost_Model_UO.py

@@ -1,20 +1,19 @@
 import sqlite3
+
+import numpy as np
 import pandas as pd
 import xgboost as xgb
 from sklearn.metrics import accuracy_score
 from sklearn.model_selection import train_test_split
 from tqdm import tqdm
-import numpy as np
-
 
 dataset = "dataset_2012-23"
 con = sqlite3.connect("../../Data/dataset.sqlite")
 data = pd.read_sql_query(f"select * from \"{dataset}\"", con, index_col="index")
 con.close()
 OU = data['OU-Cover']
 total = data['OU']
-data.drop(['Score', 'Home-Team-Win', 'TEAM_NAME', 'Date', 'TEAM_NAME.1', 'Date.1', 'OU-Cover', 'OU'], axis=1,
-          inplace=True)
+data.drop(['Score', 'Home-Team-Win', 'TEAM_NAME', 'Date', 'TEAM_NAME.1', 'Date.1', 'OU-Cover', 'OU'], axis=1, inplace=True)
 
 data['OU'] = np.asarray(total)
 data = data.values
@@ -28,12 +27,12 @@
     test = xgb.DMatrix(x_test)
 
     param = {
-        'max_depth': 6,
+        'max_depth': 20,
         'eta': 0.05,
         'objective': 'multi:softprob',
         'num_class': 3
     }
-    epochs = 300
+    epochs = 750
 
     model = xgb.train(param, train, epochs)
 
@@ -43,9 +42,9 @@
     for z in predictions:
         y.append(np.argmax(z))
 
-    acc = round(accuracy_score(y_test, y)*100, 1)
+    acc = round(accuracy_score(y_test, y) * 100, 1)
     print(f"{acc}%")
     acc_results.append(acc)
     # only save results if they are the best so far
     if acc == max(acc_results):
-        model.save_model('../../Models/XGBoost_{}%_UO-8.json'.format(acc))
+        model.save_model('../../Models/XGBoost_{}%_UO-20-01.json'.format(acc))