From a395dd9dfa3a35d8e7187855a6395a6cfc8f5a89 Mon Sep 17 00:00:00 2001 From: abyt101 Date: Thu, 20 Jun 2024 07:05:30 +0300 Subject: [PATCH 1/2] Project folder structure init --- .github/ISSUE_TEMPLATE.md | 0 .github/workflows/ci-cd.yml | 0 Dockerfile | 0 airflow/dags/backtest_dag.py | 0 airflow/dags/monitor_dag.py | 0 app/__init__.py | 0 app/main.py | 0 app/models/backtest.py | 0 app/models/user.py | 0 app/routes/auth.py | 0 app/routes/backtest.py | 0 app/routes/index.py | 0 app/services/backtest_service.py | 0 app/services/kafka_service.py | 0 app/services/mlflow_service.py | 0 app/templates/index.html | 0 config/airflow.cfg | 0 config/config.py | 0 config/kafka-config.yaml | 0 config/mlflow-config.yaml | 0 docker-compose.yml | 0 docs/DESIGN.md | 0 docs/USAGE.md | 0 notebooks/backtesting.ipynb | 0 notebooks/data_exploration.ipynb | 0 notebooks/model_training.ipynb | 0 requirements.txt | 0 scripts/backtest_runner.py | 0 scripts/data_ingestion.py | 0 scripts/model_training.py | 0 setup.py | 0 tests/conftest.py | 0 tests/integration/test_end_to_end.py | 0 tests/unit/test_backtest_service.py | 0 tests/unit/test_kafka_service.py | 0 tests/unit/test_mlflow_service.py | 0 36 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/workflows/ci-cd.yml create mode 100644 Dockerfile create mode 100644 airflow/dags/backtest_dag.py create mode 100644 airflow/dags/monitor_dag.py create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 app/models/backtest.py create mode 100644 app/models/user.py create mode 100644 app/routes/auth.py create mode 100644 app/routes/backtest.py create mode 100644 app/routes/index.py create mode 100644 app/services/backtest_service.py create mode 100644 app/services/kafka_service.py create mode 100644 app/services/mlflow_service.py create mode 100644 app/templates/index.html create mode 100644 config/airflow.cfg create mode 100644 config/config.py create mode 100644 config/kafka-config.yaml create mode 100644 config/mlflow-config.yaml create mode 100644 docker-compose.yml create mode 100644 docs/DESIGN.md create mode 100644 docs/USAGE.md create mode 100644 notebooks/backtesting.ipynb create mode 100644 notebooks/data_exploration.ipynb create mode 100644 notebooks/model_training.ipynb create mode 100644 requirements.txt create mode 100644 scripts/backtest_runner.py create mode 100644 scripts/data_ingestion.py create mode 100644 scripts/model_training.py create mode 100644 setup.py create mode 100644 tests/conftest.py create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/unit/test_backtest_service.py create mode 100644 tests/unit/test_kafka_service.py create mode 100644 tests/unit/test_mlflow_service.py diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/airflow/dags/backtest_dag.py b/airflow/dags/backtest_dag.py new file mode 100644 index 0000000..e69de29 diff --git a/airflow/dags/monitor_dag.py b/airflow/dags/monitor_dag.py new file mode 100644 index 0000000..e69de29 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models/backtest.py b/app/models/backtest.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routes/auth.py b/app/routes/auth.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routes/backtest.py b/app/routes/backtest.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routes/index.py b/app/routes/index.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/kafka_service.py b/app/services/kafka_service.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/mlflow_service.py b/app/services/mlflow_service.py new file mode 100644 index 0000000..e69de29 diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..e69de29 diff --git a/config/airflow.cfg b/config/airflow.cfg new file mode 100644 index 0000000..e69de29 diff --git a/config/config.py b/config/config.py new file mode 100644 index 0000000..e69de29 diff --git a/config/kafka-config.yaml b/config/kafka-config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/mlflow-config.yaml b/config/mlflow-config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..e69de29 diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000..e69de29 diff --git a/notebooks/backtesting.ipynb b/notebooks/backtesting.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/notebooks/data_exploration.ipynb b/notebooks/data_exploration.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/notebooks/model_training.ipynb b/notebooks/model_training.ipynb new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/scripts/backtest_runner.py b/scripts/backtest_runner.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/model_training.py b/scripts/model_training.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_kafka_service.py b/tests/unit/test_kafka_service.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_mlflow_service.py b/tests/unit/test_mlflow_service.py new file mode 100644 index 0000000..e69de29 From b788223dac80fcae7849357f466d4172b17f1592 Mon Sep 17 00:00:00 2001 From: abyt101 Date: Thu, 20 Jun 2024 07:18:45 +0300 Subject: [PATCH 2/2] Read me added --- README.md | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4ea7279..ba4d46c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,141 @@ -# backtesting-infrastructure -Crypto Trading Engineering: Scalable Backtesting Infrastructure +# Crypto Trading Backtesting + +Crypto Trading Backtesting: Scalable Backtesting Infrastructure for Cryptocurrencies + +## Overview +The Crypto Trading Backtesting project aims to provide a robust, large-scale trading data pipeline designed to help users backtest cryptocurrency trading strategies. The system uses historical candlestick data to simulate trades and evaluate strategies, providing valuable insights and reducing the risk associated with trading cryptocurrencies. + +## Features + +- **Data Ingestion** : Fetches historical candlestick data from Binance and Yahoo Finance. + +- **Backtesting Engine**: Utilizes advanced frameworks like Backtrader, Freqtrade, and Vectorbt to simulate trades based on various strategies. + +- **Metrics Calculation**: Evaluates strategies using metrics such as returns, number of trades, winning trades, losing trades, max drawdown, and Sharpe ratio. + +- **Dynamic Scene Generation**: Automates backtesting runs with customizable parameters. + +- **MLOps Integration**: Incorporates tools like Apache Airflow, Kafka, MLflow, and CML for seamless pipeline automation and monitoring. + +- **Frontend Interface**: Provides a user-friendly interface for running backtests and viewing results. + +## Project Structure + +``` +crypto-trading-backtesting/ +│ +├── .github/ +│ ├── workflows/ +│ │ └── ci-cd.yml +│ └── ISSUE_TEMPLATE.md +│ +├── airflow/ +│ ├── dags/ +│ │ ├── backtest_dag.py +│ │ └── monitor_dag.py +│ └── plugins/ +│ +├── app/ +│ ├── __init__.py +│ ├── main.py +│ ├── models/ +│ │ ├── backtest.py +│ │ └── user.py +│ ├── routes/ +│ │ ├── auth.py +│ │ ├── backtest.py +│ │ └── index.py +│ ├── services/ +│ │ ├── backtest_service.py +│ │ ├── kafka_service.py +│ │ └── mlflow_service.py +│ └── templates/ +│ └── index.html +│ +├── config/ +│ ├── airflow.cfg +│ ├── config.py +│ ├── kafka-config.yaml +│ └── mlflow-config.yaml +│ +├── data/ +│ ├── raw/ +│ ├── processed/ +│ └── backtests/ +│ +├── docs/ +│ ├── README.md +│ ├── DESIGN.md +│ └── USAGE.md +│ +├── notebooks/ +│ ├── data_exploration.ipynb +│ ├── backtesting.ipynb +│ └── model_training.ipynb +│ +├── scripts/ +│ ├── backtest_runner.py +│ ├── data_ingestion.py +│ └── model_training.py +│ +├── tests/ +│ ├── unit/ +│ │ ├── test_backtest_service.py +│ │ ├── test_kafka_service.py +│ │ └── test_mlflow_service.py +│ ├── integration/ +│ │ ├── test_end_to_end.py +│ └── conftest.py +│ +├── Dockerfile +├── docker-compose.yml +├── requirements.txt +└── setup.py +``` + +### Installation + +1. **Clone the repository:** + + ```sh + git clone git@github.com:your-username/crypto_trading_backtesting.git + cd crypto_trading_backtesting + ``` + +2. **Set up a virtual environment:** + + ```sh + python3 -m venv venv + source venv/bin/activate # On Windows, use `venv\Scripts\activate` + ``` +3. **Install Requirements:** + ```sh + pip install -r requirements.txt + ``` +4. **Run Streamlit interface** + ```sh + streamlit run src/frontend/app.py + ``` + +## Usage +1. **Provide input descriptions** of the trading strategies, including scenarios and expected outputs. + +2. **Run the backtesting system** to simulate trades and evaluate strategies. +3. **Review the generated metrics** to assess the performance of different trading strategies. +4. **Select the desired strategies** for further use or real-time trading. + +## License + +This project is licensed under the MIT License. + + +## Contributors + +- [@abyt101](https://github.com/AbYT101) - Abraham Teka +- Temesgen Gebreabzgi +- Selamawit Tibebu +- Dereje Hinsermu + +## Challenge by + +![10 Academy](https://static.wixstatic.com/media/081e5b_5553803fdeec4cbb817ed4e85e1899b2~mv2.png/v1/fill/w_246,h_106,al_c,q_85,usm_0.66_1.00_0.01,enc_auto/10%20Academy%20FA-02%20-%20transparent%20background%20-%20cropped.png) \ No newline at end of file