xgb模型取消gpu模式:部分机器或集群可能由于没有gpu导致运行失败.

Author: enjoysport2022

autox/models/classifier.py

@@ -33,7 +33,7 @@ def __init__(self, params=None, n_fold=10):
             'reg_alpha': 40,
             'reg_lambda': 18,
             'min_child_weight': 16,
-            'tree_method': 'gpu_hist'
+            # 'tree_method': 'gpu_hist'
         }
         if params is not None:
             self.params_ = params
@@ -56,7 +56,7 @@ def objective(trial):
                 'reg_lambda': trial.suggest_int('reg_lambda', 5, 100),
                 'min_child_weight': trial.suggest_int('min_child_weight', 5, 20),
             }
-            reg = xgb.XGBClassifier(tree_method='gpu_hist', **param_grid)
+            reg = xgb.XGBClassifier(**param_grid) # tree_method='gpu_hist',
             reg.fit(X_train, y_train,
                     eval_set=[(X_valid, y_valid)], eval_metric='auc',
                     verbose=False)
@@ -79,7 +79,7 @@ def objective(trial):
 
         self.params_ = trial.params
         self.params_['eta'] = 0.01
-        self.params_['tree_method'] = 'gpu_hist'
+        # self.params_['tree_method'] = 'gpu_hist'
 
     def fit(self, X, y, tuning=True, Debug=False):
         log(X.shape)

autox/models/regressor.py

@@ -139,7 +139,7 @@ def __init__(self, metric, params=None, n_fold=10):
             'reg_alpha': 40,
             'reg_lambda': 18,
             'min_child_weight': 16,
-            'tree_method': 'gpu_hist',
+            # 'tree_method': 'gpu_hist',
             'verbose_eval': 100
         }
         if params is not None:
@@ -165,7 +165,7 @@ def objective(trial):
                 'reg_lambda': trial.suggest_int('reg_lambda', 5, 100),
                 'min_child_weight': trial.suggest_int('min_child_weight', 5, 20),
             }
-            reg = xgb.XGBRegressor(tree_method='gpu_hist', **param_grid)
+            reg = xgb.XGBRegressor(**param_grid) # tree_method='gpu_hist',
             reg.fit(X_train, y_train,
                     eval_set=[(X_valid, y_valid)], eval_metric=metric,
                     verbose=False)
@@ -192,7 +192,7 @@ def objective(trial):
 
         self.params_ = trial.params
         self.params_['eta'] = 0.01
-        self.params_['tree_method'] = 'gpu_hist'
+        # self.params_['tree_method'] = 'gpu_hist'
 
     def fit(self, X, y, tuning=True, Debug=False, log1p=True):
         log(X.shape)

autox/run.py

@@ -3,9 +3,12 @@
 
 ap = argparse.ArgumentParser(description='run_autox.py')
 ap.add_argument('path_input', nargs=1, action="store", type=str)
+ap.add_argument('path_output', nargs=1, action="store", type=str)
 pa = ap.parse_args()
 path_input = pa.path_input[0]
+path_output = pa.path_output[0]
 print("path_input: ", path_input)
+print("path_output: ", path_output)
 
 
 
@@ -18,4 +21,4 @@
 
 sub = autox.get_submit()
 
-sub.to_csv(f"autox_{data_name}_oneclick.csv", index = False)
+sub.to_csv(f"{path_output}/autox_{data_name}_oneclick.csv", index = False)

submit.py

@@ -13,7 +13,7 @@ def get_run_sh(hdfs_env, hdfs_input, path_src, time_str, app_name, mem):
 
     # python_cmd = 'python -u main.py {} {} | tee {}/log.txt'.format(path_input, path_output, path_output)
     # python_cmd = 'python -u fengdian_ensemble.py {} -1 {} | tee {}/log.txt'.format(path_output, path_input, path_output)
-    python_cmd = f'python -u run.py {path_input} | tee {path_output}/kaggle_house_price.log'
+    python_cmd = f'python -u run.py {path_input} {path_output} | tee {path_output}/kaggle_house_price.log'
 
     cmd_hdfs_get = 'hdfs dfs -get {}/{}/ .'.format(hdfs_input, name_output)
     file = open('cmd_hdfs_get.sh', 'a')
@@ -63,7 +63,7 @@ def get_run_sh(hdfs_env, hdfs_input, path_src, time_str, app_name, mem):
     s += ['echo "[+] output......"']
     s += ['mkdir {}'.format(name_output)]
     s += ['hdfs dfs -put -f -p {}/ {}/'.format(name_output, hdfs_input)]
-    s += ['hdfs dfs -put -f src.tar.gz {}/{}/'.format(hdfs_input, name_output)]
+    s += ['hdfs dfs -put -f autox.tar.gz {}/{}/'.format(hdfs_input, name_output)]
     s += ['hdfs dfs -put -f run.sh {}/{}/'.format(hdfs_input, name_output)]
     s += ['hdfs dfs -put -f yarn.sh {}/{}/'.format(hdfs_input, name_output)]
     s += ['hdfs dfs -put -f submit.py {}/{}/'.format(hdfs_input, name_output)]
@@ -101,7 +101,7 @@ def get_yarn_sh(appname, mem=65536, queue='pico'):
     s += ['  --num_containers=1 \\']
     s += ['  --shell_env HADOOP_USER_NAME=`whoami`\\']
     s += ['  --shell_env WEBHDFS_USER=`whoami` \\']
-    s += ['  --file src.tar.gz \\']
+    s += ['  --file autox.tar.gz \\']
     s += ['  --file submit.py \\']
     s += ['  --file run.sh \\']
     s += ['  --file yarn.sh']
@@ -159,7 +159,7 @@ def tar_src(path_src):
     return path_src
 
 
-path_src = 'src'
+path_src = 'autox'
 path_src = tar_src(path_src)
 if path_src is None:
     print('[+] src not exist!')