diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dd02ce0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10-bookworm + +COPY . /app +WORKDIR /app + +EXPOSE 48888 +RUN pip install -r requirements.txt + +ENTRYPOINT ["python", "serve.py"] + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5103598 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ + +# Flask API Server for PTN Outputs + +## Overview +This Flask server is configured to serve the outputs of the [PTN repo (Bittensor subnet 8)](https://github.com/taoshidev/proprietary-trading-network/blob/main/docs/validator.md), which generates JSON files accessible via this API. The Flask code can easily be modified to serve outputs from any other Bittensor subnet. + +## Security Warning +The API uses a simple token-based authentication system. The default API key is set to "xxxx". **Change this default API key before deploying in a production environment to prevent unauthorized access.** + +## Configuration +### Changing the API Key +To enhance the security of your API, change the default API key in the `accessible_api_keys` list in `serve.py` to a more secure key. + +### Making Server Accessible +By default, the server binds to `127.0.0.1` which only allows local requests. To allow access from any IP address, bind to `0.0.0.0`: +```python +serve(app, host="0.0.0.0", port=48888) +``` + +### Launching the server +On your validator, clone this repo in the same directory that the `proprietary-trading-network` repo is in, make desired edits, and then run +```bash +pm2 start serve.py --name serve +``` + +## Security Considerations +### Rate Limiting +Consider implementing rate limiting using out of the box Flask extensions like `Flask-Limiter` or custom implementations to prevent abuse and ensure fair use of the API. + +### HTTPS +Deploy the Flask application over HTTPS in production to encrypt data in transit. This is typically done by placing the Flask application behind a reverse proxy that handles SSL/TLS termination. + +### Firewall Configuration +Configure firewall rules to only allow traffic on necessary ports from trusted IP addresses. + +## Usage with curl +Example `curl` commands to interact with the Flask server. Replace `` with your validator's IP address and `xxxx` with your API key that you hardcode in `serve.py` + +### Get Miner Positions with curl +```bash +curl -X GET http://:48888/miner-positions -H "Content-Type: application/json" -d '{"api_key": "xxxx"}' -o miner_positions.json +``` + +### Get Validator Checkpoint with curl +```bash +curl -X GET http://:48888/validator-checkpoint -H "Content-Type: application/json" -d '{"api_key": "xxxx"}' -o validator_checkpoint.json +``` + +## Usage with Python +Example python code to interact with the Flask server. + +### Get Validator Checkpoint with python + +```python +import requests +import json + +url = 'https://example.com/validator-checkpoint' +api_key = 'abcdefg' + +data = { +'api_key': api_key +} +json_data = json.dumps(data) +headers = { +'Content-Type': 'application/json', +} +test = requests.get(url, data=json_data, headers=headers) +print(test) +with open('validator_checkpoint.json', 'w') as f: + f.write(json.dumps(test.json())) +# print(json.loads(test.json())) + +``` + + +## Final Notes +This Flask server setup provides a simple template for serving API requests. If not using the Request Network, ensure all security measures are in place before deploying to a live environment. + +The Request Network is a Taoshi product which serves subnet data while handling security, rate limiting, data customization, and provide a polished customer-facing and validator setup UI. + diff --git a/requirements.txt b/requirements.txt index 8642e2b..3c94e51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask -waitress \ No newline at end of file +waitress +requests diff --git a/serve.py b/serve.py index 9977673..5296d93 100644 --- a/serve.py +++ b/serve.py @@ -1,114 +1,232 @@ -import hashlib -import json +import sys -from flask import Flask, jsonify +from flask import Flask, jsonify, request, Response import os - -from datetime import datetime, timezone +import time +import json from waitress import serve app = Flask(__name__) +accessible_api_keys = [ + 'xxxx' +] + + +def get_api_key(): + # Get the API key from the query parameters or request headers + if "api_key" in request.json: + api_key = request.json["api_key"] + else: + api_key = request.headers.get('Authorization') + if api_key: + api_key = api_key.split(' ')[1] # Remove 'Bearer ' prefix + return api_key + + +def get_file(f, attempts=3, binary=False): + file_path = os.path.abspath(os.path.join(path, f)) + if not os.path.exists(file_path): + return None + + for attempt_number in range(attempts): + try: + if binary: + with open(file_path, 'rb') as f: + data = f.read() + else: + with open(file_path, "r") as file: + data = json.load(file) + return data + except json.JSONDecodeError as e: + if attempt_number == attempts - 1: + print(f"serve.py Failed to decode JSON after multiple attempts: {e}") + raise + else: + print(f"serve.py Attempt {attempt_number + 1} failed with JSONDecodeError, retrying...") + time.sleep(1) # Wait before retrying + except Exception as e: + print(f"serve.py Unexpected error reading file: {e}") + raise + +# Endpoint to read and serve JSON data from outputs.json +@app.route("/miner-positions", methods=["GET"]) +def get_miner_positions(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + # Get the 'tier' query parameter from the request + tier = request.args.get('tier') + is_gz_data = tier is not None + + if is_gz_data: + # Validate the 'tier' parameter + if tier not in ['0', '30', '50', '100']: + return jsonify({'error': 'Invalid tier value. Allowed values are 0, 30, 50, or 100'}), 400 + + # Construct the relative path based on the specified tier + f = f"outputs/tiered_positions/{tier}/output.json.gz" + else: + # If 'tier' parameter is not provided, return the default output.json + f = "outputs/output.json" + + # Attempt to retrieve the file + data = get_file(f, binary=is_gz_data) + + if data is None: + return f"{f} not found", 404 + if is_gz_data: + return Response(data, content_type='application/json', headers={ + 'Content-Encoding': 'gzip' + }) + return jsonify(data) + + +@app.route("/miner-positions/", methods=["GET"]) +def get_miner_positions_unique(minerid): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "outputs/output.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + # Filter the data for the specified miner ID + filtered_data = data.get(minerid, None) + + if filtered_data is None: + return jsonify({'error': 'Miner ID not found'}), 404 + + return jsonify(filtered_data) + +# Endpoint to read and serve JSON data from outputs.json +@app.route("/miner-hotkeys", methods=["GET"]) +def get_miner_hotkeys(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "outputs/output.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + miner_hotkeys = list(data.keys()) + + if len(miner_hotkeys) == 0: + return f"{f} not found", 404 + else: + return jsonify(miner_hotkeys) + +# serve miner positions v2 now named validator checkpoint +@app.route("/validator-checkpoint", methods=["GET"]) +def get_validator_checkpoint(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "../runnable/validator_checkpoint.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) + +# serve miner positions v2 now named validator checkpoint +@app.route("/statistics", methods=["GET"]) +def get_validator_checkpoint_statistics(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "../runnable/minerstatistics.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) + +# serve miner positions v2 now named validator checkpoint +@app.route("/statistics//", methods=["GET"]) +def get_validator_checkpoint_statistics_unique(minerid): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "../runnable/minerstatistics.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + data_summary: list = data.get("data", None) + for element in data_summary: + if element.get("hotkey", None) == minerid: + return jsonify(element) + + return jsonify({'error': 'Miner ID not found'}), 404 + + +@app.route("/eliminations", methods=["GET"]) +def get_eliminations(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "eliminations.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) + + +@app.route("/miner-copying", methods=["GET"]) +def get_miner_copying(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "miner_copying.json" + data = get_file(f) -# Endpoint to read and serve JSON data from cmw.json -@app.route("/cmw", methods=["GET"]) -def get_cmw_data(): - cmw_json_path = os.path.abspath(os.path.join(path, "outputs/cmw.json")) - if os.path.exists(cmw_json_path): - with open(cmw_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{cmw_json_path} not found", 404 - - -# Endpoint to read and serve JSON data from latest_predictions.json -@app.route("/predictions", methods=["GET"]) -def get_predictions_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "outputs/latest_predictions.json") - ) - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{predictions_json_path} not found", 404 - - -@app.route("/weights", methods=["GET"]) -def get_weights_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "weights/valiweights.json") - ) - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{predictions_json_path} not found", 404 - - -@app.route("/unique-predictions", methods=["GET"]) -def get_unique_predictions_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "outputs/latest_predictions.json") - ) - results = {} - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - results = json.loads(file.read()) - - predictions = [] - preds_to_miners = {} - - ts_list = [] - start_ms = results["BTCUSD-5m"][0]["start"] - - ts_list.append(start_ms) - - for i in range(1, 100): - ts_list.append(start_ms + i * 60000 * 5) - - for ts in ts_list: - print(datetime.utcfromtimestamp(ts / 1000).replace(tzinfo=timezone.utc)) - - for v in results["BTCUSD-5m"]: - hashed_preds = hashlib.sha256(str(v["predictions"]).encode()).hexdigest() - if hashed_preds not in predictions: - predictions.append(hashed_preds) - preds_to_miners[v["miner_uid"]] = v["predictions"] - - return_results_dict = {i: {"timestamp": ts_list[i]} for i in range(0, 100)} - - for key, value in preds_to_miners.items(): - for i, v in enumerate(value): - return_results_dict[i][key] = v - - return jsonify({"unique_predictions": [return_results_dict[i] for i in range(0, 100)]}) - - -@app.route("/latest-cmw", methods=["GET"]) -def get_latest_cmw(): - directory = os.path.abspath( - os.path.join(path, "backups/") - ) - # Get list of all files in the directory - files = os.listdir(directory) - # Filter out directories, leave only files - files = [os.path.join(directory, file) for file in files if os.path.isfile(os.path.join(directory, file))] - # Get the latest file based on creation time - latest_file = max(files, key=os.path.getctime) - - if os.path.exists(latest_file): - with open(latest_file, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{latest_file} not found", 404 + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) if __name__ == "__main__": - path = "time-series-prediction-subnet/validation/" - serve(app, host="0.0.0.0", port=80) + # sys.argv[0] is the script name itself + # Arguments start from sys.argv[1] + if len(sys.argv) > 1: + path = sys.argv[1] + else: + path = "../proprietary-trading-network/validation/" + print(path) + serve(app, host="127.0.0.1", port=48888)