Merge upstream changes

Author: notadamking

README.md

@@ -3,8 +3,9 @@
 [![Build Status](https://travis-ci.org/notadamking/RLTrader.svg?branch=master)](https://travis-ci.org/notadamking/RLTrader)
 [![Contributions Welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg)](http://makeapullrequest.com)
 [![GPL License](https://img.shields.io/github/license/notadamking/RLTrader.svg?color=brightgreen)](https://opensource.org/licenses/GPL-3.0/)
+[![Discord](https://img.shields.io/discord/592446624882491402.svg?color=brightgreen)](https://discord.gg/ZZ7BGWh)
 [![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
-[![Github Stars](https://img.shields.io/github/stars/notadamking/RLTrader.svg)](https://github.com/notadamking/RLTrader)
+[![Github Release](https://img.shields.io/github/release/notadamking/RLTrader.svg)](https://github.com/notadamking/RLTrader)
 
 In this series of articles, we've created and optimized a Bitcoin trading agent to be highly profitable using deep reinforcement learning.
 
@@ -63,17 +64,17 @@ python ./cli.py --help
 or simply run the project with default options:
 
 ```bash
-python ./cli.py optimize-train-test
+python ./cli.py optimize
 ```
 
 If you have a standard set of configs you want to run the trader against, you can specify a config file to load configuration from. Rename config/config.ini.dist to config/config.ini and run
 
 ```bash
-python ./cli.py --from-config config/config.ini optimize-train-test
+python ./cli.py --from-config config/config.ini optimize
 ```
 
 ```bash
-python ./cli.py optimize-train-test
+python ./cli.py optimize
 ```
 
 ### Testing with vagrant
@@ -92,7 +93,7 @@ Note: With vagrant you cannot take full advantage of your GPU, so is mainly for
 If you want to run everything within a docker container, then just use:
 
 ```bash
-./run-with-docker (cpu|gpu) (yes|no) optimize-train-test
+./run-with-docker (cpu|gpu) (yes|no) optimize
 ```
 
 - cpu - start the container using CPU requirements
@@ -101,7 +102,7 @@ If you want to run everything within a docker container, then just use:
   Note: in case using yes as second argument, use
 
 ```bash
-python ./ cli.py --params-db-path "postgres://rl_trader:rl_trader@localhost" optimize-train-test
+python ./ cli.py --params-db-path "postgres://rl_trader:rl_trader@localhost" optimize
 ```
 
 The database and it's data are pesisted under `data/postgres` locally.

cli.py

@@ -1,6 +1,5 @@
 import numpy as np
-
-from deco import concurrent
+import multiprocessing
 
 from lib.RLTrader import RLTrader
 from lib.cli.RLTraderCLI import RLTraderCLI
@@ -12,27 +11,34 @@
 args = trader_cli.get_args()
 
 
-@concurrent(processes=args.parallel_jobs)
-def run_concurrent_optimize(trader: RLTrader, args):
-    trader.optimize(args.trials, args.trials, args.parallel_jobs)
+def run_concurrent_optimize():
+    trader = RLTrader(**vars(args))
+    trader.optimize(args.trials)
+
+
+def concurrent_optimize():
+    processes = []
+    for i in range(args.parallel_jobs):
+        processes.append(multiprocessing.Process(target=run_concurrent_optimize, args=()))
+
+    print(processes)
+
+    for p in processes:
+        p.start()
+
+    for p in processes:
+        p.join()
 
 
 if __name__ == '__main__':
     logger = init_logger(__name__, show_debug=args.debug)
     trader = RLTrader(**vars(args), logger=logger)
 
     if args.command == 'optimize':
-        run_concurrent_optimize(trader, args)
+        concurrent_optimize()
     elif args.command == 'train':
         trader.train(n_epochs=args.epochs)
     elif args.command == 'test':
         trader.test(model_epoch=args.model_epoch, should_render=args.no_render)
-    elif args.command == 'optimize-train-test':
-        run_concurrent_optimize(trader, args)
-        trader.train(
-            n_epochs=args.train_epochs,
-            test_trained_model=args.no_test,
-            render_trained_model=args.no_render
-        )
     elif args.command == 'update-static-data':
         download_data_async()

lib/RLTrader.py

@@ -230,23 +230,28 @@ def test(self, model_epoch: int = 0, should_render: bool = True):
 
         del train_provider
 
-        test_env = SubprocVecEnv([make_env(test_provider, i) for i in range(self.n_envs)])
+        test_env = DummyVecEnv([make_env(test_provider, i) for i in range(1)])
 
         model_path = path.join('data', 'agents', f'{self.study_name}__{model_epoch}.pkl')
         model = self.Model.load(model_path, env=test_env)
 
         self.logger.info(f'Testing model ({self.study_name}__{model_epoch})')
 
+        zero_completed_obs = np.zeros((self.n_envs,) + test_env.observation_space.shape)
+        zero_completed_obs[0, :] = test_env.reset()
+
         state = None
-        obs, rewards = test_env.reset(), []
+        rewards = []
 
         for _ in range(len(test_provider.data_frame)):
-            action, state = model.predict(obs, state=state)
-            obs, reward, _, __ = test_env.step(action)
+            action, state = model.predict(zero_completed_obs, state=state)
+            obs, reward, _, __ = test_env.step([action])
+
+            zero_completed_obs[0, :] = obs
 
             rewards.append(reward)
 
-            if should_render and self.n_envs == 1:
+            if should_render:
                 test_env.render(mode='human')
 
         self.logger.info(

lib/cli/RLTraderCLI.py

@@ -44,12 +44,6 @@ def __init__(self):
 
         subparsers = self.parser.add_subparsers(help='Command', dest="command")
 
-        opt_train_test_parser = subparsers.add_parser('optimize-train-test', description='Optimize train and test')
-        opt_train_test_parser.add_argument('--trials', type=int, default=20, help='Number of trials')
-        opt_train_test_parser.add_argument('--train-epochs', type=int, default=10, help='Train for how many epochs')
-        opt_train_test_parser.add_argument('--no-render', action='store_false', help='Should render the model')
-        opt_train_test_parser.add_argument('--no-test', action='store_false', help='Should test the model')
-
         optimize_parser = subparsers.add_parser('optimize', description='Optimize model parameters')
         optimize_parser.add_argument('--trials', type=int, default=1, help='Number of trials')
 

optimize.py

@@ -15,15 +15,17 @@ def optimize_code(params):
     n_process = multiprocessing.cpu_count()
     params = {'n_cpu': n_process}
 
-    processes = []
-    for i in range(n_process):
-        processes.append(multiprocessing.Process(target=optimize_code, args=(params,)))
+    # processes = []
+    # for i in range(n_process):
+    #     processes.append(multiprocessing.Process(target=optimize_code, args=(params,)))
 
-    for p in processes:
-        p.start()
+    # for p in processes:
+    #     p.start()
 
-    for p in processes:
-        p.join()
+    # for p in processes:
+    #     p.join()
 
     trader = RLTrader(**params)
-    trader.train(test_trained_model=True, render_trained_model=True)
+    # trader.train(test_trained_model=True, render_trained_model=True)
+
+    trader.test(model_epoch=10)

requirements.base.txt

@@ -10,5 +10,4 @@ statsmodels==0.10.0rc2
 empyrical
 ccxt
 psycopg2
-deco
 configparser