From 04faec6f4a584874daaa7c9e3e36ed8734ccbadb Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Fri, 8 Mar 2024 22:18:46 -0800 Subject: [PATCH 01/17] serve pn data --- serve.py | 103 ++++--------------------------------------------------- 1 file changed, 6 insertions(+), 97 deletions(-) diff --git a/serve.py b/serve.py index 9977673..ce190ae 100644 --- a/serve.py +++ b/serve.py @@ -1,114 +1,23 @@ -import hashlib -import json - from flask import Flask, jsonify import os -from datetime import datetime, timezone - from waitress import serve app = Flask(__name__) # Endpoint to read and serve JSON data from cmw.json -@app.route("/cmw", methods=["GET"]) +@app.route("/miner-positions", methods=["GET"]) def get_cmw_data(): - cmw_json_path = os.path.abspath(os.path.join(path, "outputs/cmw.json")) - if os.path.exists(cmw_json_path): - with open(cmw_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{cmw_json_path} not found", 404 - - -# Endpoint to read and serve JSON data from latest_predictions.json -@app.route("/predictions", methods=["GET"]) -def get_predictions_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "outputs/latest_predictions.json") - ) - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{predictions_json_path} not found", 404 - - -@app.route("/weights", methods=["GET"]) -def get_weights_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "weights/valiweights.json") - ) - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - data = file.read() - return jsonify(data) - else: - return f"{predictions_json_path} not found", 404 - - -@app.route("/unique-predictions", methods=["GET"]) -def get_unique_predictions_data(): - predictions_json_path = os.path.abspath( - os.path.join(path, "outputs/latest_predictions.json") - ) - results = {} - if os.path.exists(predictions_json_path): - with open(predictions_json_path, "r") as file: - results = json.loads(file.read()) - - predictions = [] - preds_to_miners = {} - - ts_list = [] - start_ms = results["BTCUSD-5m"][0]["start"] - - ts_list.append(start_ms) - - for i in range(1, 100): - ts_list.append(start_ms + i * 60000 * 5) - - for ts in ts_list: - print(datetime.utcfromtimestamp(ts / 1000).replace(tzinfo=timezone.utc)) - - for v in results["BTCUSD-5m"]: - hashed_preds = hashlib.sha256(str(v["predictions"]).encode()).hexdigest() - if hashed_preds not in predictions: - predictions.append(hashed_preds) - preds_to_miners[v["miner_uid"]] = v["predictions"] - - return_results_dict = {i: {"timestamp": ts_list[i]} for i in range(0, 100)} - - for key, value in preds_to_miners.items(): - for i, v in enumerate(value): - return_results_dict[i][key] = v - - return jsonify({"unique_predictions": [return_results_dict[i] for i in range(0, 100)]}) - - -@app.route("/latest-cmw", methods=["GET"]) -def get_latest_cmw(): - directory = os.path.abspath( - os.path.join(path, "backups/") - ) - # Get list of all files in the directory - files = os.listdir(directory) - # Filter out directories, leave only files - files = [os.path.join(directory, file) for file in files if os.path.isfile(os.path.join(directory, file))] - # Get the latest file based on creation time - latest_file = max(files, key=os.path.getctime) - - if os.path.exists(latest_file): - with open(latest_file, "r") as file: + output_json_path = os.path.abspath(os.path.join(path, "outputs/output.json")) + if os.path.exists(output_json_path): + with open(output_json_path, "r") as file: data = file.read() return jsonify(data) else: - return f"{latest_file} not found", 404 + return f"{output_json_path} not found", 404 if __name__ == "__main__": - path = "time-series-prediction-subnet/validation/" + path = "../prop-net/validation/" serve(app, host="0.0.0.0", port=80) From 7a94c862eabb597e0e94c8945e64a58e3fdf4538 Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Sat, 9 Mar 2024 20:28:07 -0800 Subject: [PATCH 02/17] update comment --- serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serve.py b/serve.py index ce190ae..01bb492 100644 --- a/serve.py +++ b/serve.py @@ -6,7 +6,7 @@ app = Flask(__name__) -# Endpoint to read and serve JSON data from cmw.json +# Endpoint to read and serve JSON data from outputs.json @app.route("/miner-positions", methods=["GET"]) def get_cmw_data(): output_json_path = os.path.abspath(os.path.join(path, "outputs/output.json")) From 80e9adb02a6002e07bab1dcc1340adf0af73782e Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Wed, 13 Mar 2024 13:24:27 -0700 Subject: [PATCH 03/17] adding elims and copying endpoints --- requirements.txt | 3 +- serve.py | 84 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8642e2b..3c94e51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ flask -waitress \ No newline at end of file +waitress +requests diff --git a/serve.py b/serve.py index 01bb492..b4d90f9 100644 --- a/serve.py +++ b/serve.py @@ -1,21 +1,93 @@ -from flask import Flask, jsonify +from flask import Flask, jsonify, request import os +import requests from waitress import serve app = Flask(__name__) +accessible_api_keys = [ + 'xxxx' +] -# Endpoint to read and serve JSON data from outputs.json -@app.route("/miner-positions", methods=["GET"]) -def get_cmw_data(): - output_json_path = os.path.abspath(os.path.join(path, "outputs/output.json")) + +def get_api_key(): + # Get the API key from the query parameters or request headers + if "api_key" in request.json: + api_key = request.json["api_key"] + else: + api_key = request.headers.get('Authorization') + if api_key: + api_key = api_key.split(' ')[1] # Remove 'Bearer ' prefix + return api_key + + +def get_file(f): + output_json_path = os.path.abspath(os.path.join(path, f)) if os.path.exists(output_json_path): with open(output_json_path, "r") as file: data = file.read() + return data + else: + return None + + +# Endpoint to read and serve JSON data from outputs.json +@app.route("/miner-positions", methods=["GET"]) +def get_miner_positions(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + else: + print("api key received.") + + f = "outputs/output.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: return jsonify(data) + + +@app.route("/eliminations", methods=["GET"]) +def get_eliminations(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + else: + print("api key received.") + + f = "eliminations.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 else: - return f"{output_json_path} not found", 404 + return jsonify(data) + + +@app.route("/miner-copying", methods=["GET"]) +def get_miner_copying(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + else: + print("api key received.") + + f = "miner_copying.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) if __name__ == "__main__": From 776c1a94e12b8f943a67c4bdd0b3799b9d50d29e Mon Sep 17 00:00:00 2001 From: Thomas Dougherty Date: Thu, 21 Mar 2024 17:08:09 -0700 Subject: [PATCH 04/17] Cleaning up formatting on json --- serve.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serve.py b/serve.py index b4d90f9..0ad2f24 100644 --- a/serve.py +++ b/serve.py @@ -1,6 +1,7 @@ from flask import Flask, jsonify, request import os import requests +import json from waitress import serve @@ -26,7 +27,7 @@ def get_file(f): output_json_path = os.path.abspath(os.path.join(path, f)) if os.path.exists(output_json_path): with open(output_json_path, "r") as file: - data = file.read() + data = json.load(file) return data else: return None From 816bc5807d79f18ae3db737606c577081d14f342 Mon Sep 17 00:00:00 2001 From: Thomas Dougherty Date: Thu, 21 Mar 2024 17:22:39 -0700 Subject: [PATCH 05/17] Cleaning up path to outputs --- serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serve.py b/serve.py index 0ad2f24..34942e5 100644 --- a/serve.py +++ b/serve.py @@ -92,5 +92,5 @@ def get_miner_copying(): if __name__ == "__main__": - path = "../prop-net/validation/" + path = "proprietary-trading-network/validation/" serve(app, host="0.0.0.0", port=80) From e859a28aa53662f60bcccf62ff2fffdac7dc139c Mon Sep 17 00:00:00 2001 From: Thomas Dougherty Date: Thu, 21 Mar 2024 17:28:02 -0700 Subject: [PATCH 06/17] Cleaning up port and path to output files --- serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serve.py b/serve.py index 34942e5..ae51270 100644 --- a/serve.py +++ b/serve.py @@ -93,4 +93,4 @@ def get_miner_copying(): if __name__ == "__main__": path = "proprietary-trading-network/validation/" - serve(app, host="0.0.0.0", port=80) + serve(app, host="0.0.0.0", port=48888) From 9c0f418e876a2216d91d9a6e008c65c571f15b61 Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Sun, 24 Mar 2024 22:00:45 -0700 Subject: [PATCH 07/17] add validator checkpoint endpoint --- serve.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/serve.py b/serve.py index ae51270..c48d658 100644 --- a/serve.py +++ b/serve.py @@ -53,6 +53,26 @@ def get_miner_positions(): return jsonify(data) +# serve miner positions v2 now named validator checkpoint +@app.route("/validator-checkpoint", methods=["GET"]) +def get_miner_positions(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + else: + print("api key received.") + + f = "runnable/validator_checkpoint.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) + + @app.route("/eliminations", methods=["GET"]) def get_eliminations(): api_key = get_api_key() From 4bc3bd84502b748350c15d395fb2ecde736293db Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Sun, 24 Mar 2024 22:05:01 -0700 Subject: [PATCH 08/17] update path ' --- serve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serve.py b/serve.py index c48d658..d440663 100644 --- a/serve.py +++ b/serve.py @@ -112,5 +112,5 @@ def get_miner_copying(): if __name__ == "__main__": - path = "proprietary-trading-network/validation/" - serve(app, host="0.0.0.0", port=48888) + path = "~/proprietary-trading-network/validation/" + serve(app, host="127.0.0.1", port=48888) From 578b09f408490bbad81f9c3439d2668b8e408593 Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Sun, 24 Mar 2024 22:07:44 -0700 Subject: [PATCH 09/17] update validator checkpoint function name --- serve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serve.py b/serve.py index d440663..d49dae9 100644 --- a/serve.py +++ b/serve.py @@ -55,7 +55,7 @@ def get_miner_positions(): # serve miner positions v2 now named validator checkpoint @app.route("/validator-checkpoint", methods=["GET"]) -def get_miner_positions(): +def get_validator_checkpoint(): api_key = get_api_key() # Check if the API key is valid From e642911f82a4e902e5099bf26bea088cde2c16cc Mon Sep 17 00:00:00 2001 From: taoshidev1 Date: Sun, 24 Mar 2024 22:23:07 -0700 Subject: [PATCH 10/17] allow arg and update path --- serve.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/serve.py b/serve.py index d49dae9..9fe6bd8 100644 --- a/serve.py +++ b/serve.py @@ -1,3 +1,5 @@ +import sys + from flask import Flask, jsonify, request import os import requests @@ -64,7 +66,7 @@ def get_validator_checkpoint(): else: print("api key received.") - f = "runnable/validator_checkpoint.json" + f = "../runnable/validator_checkpoint.json" data = get_file(f) if data is None: @@ -112,5 +114,11 @@ def get_miner_copying(): if __name__ == "__main__": - path = "~/proprietary-trading-network/validation/" + # sys.argv[0] is the script name itself + # Arguments start from sys.argv[1] + if len(sys.argv) > 1: + path = sys.argv[1] + else: + path = "../proprietary-trading-network/validation/" + print(path) serve(app, host="127.0.0.1", port=48888) From d49ae0c3ba84ef8e6f16aed69bff8a039097d5e5 Mon Sep 17 00:00:00 2001 From: jbonilla-tao <161871533+jbonilla-tao@users.noreply.github.com> Date: Tue, 30 Apr 2024 20:19:43 -0400 Subject: [PATCH 11/17] minimize error log spamming --- serve.py | 157 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 80 insertions(+), 77 deletions(-) diff --git a/serve.py b/serve.py index 9fe6bd8..4a284f5 100644 --- a/serve.py +++ b/serve.py @@ -2,7 +2,7 @@ from flask import Flask, jsonify, request import os -import requests +import time import json from waitress import serve @@ -10,115 +10,118 @@ app = Flask(__name__) accessible_api_keys = [ - 'xxxx' + 'xxxx' ] def get_api_key(): - # Get the API key from the query parameters or request headers - if "api_key" in request.json: - api_key = request.json["api_key"] - else: - api_key = request.headers.get('Authorization') - if api_key: - api_key = api_key.split(' ')[1] # Remove 'Bearer ' prefix - return api_key - - -def get_file(f): - output_json_path = os.path.abspath(os.path.join(path, f)) - if os.path.exists(output_json_path): - with open(output_json_path, "r") as file: - data = json.load(file) - return data - else: - return None - + # Get the API key from the query parameters or request headers + if "api_key" in request.json: + api_key = request.json["api_key"] + else: + api_key = request.headers.get('Authorization') + if api_key: + api_key = api_key.split(' ')[1] # Remove 'Bearer ' prefix + return api_key + + +def get_file(f, attempts=3): + output_json_path = os.path.abspath(os.path.join(path, f)) + if not os.path.exists(output_json_path): + return None + + for attempt_number in range(attempts): + try: + with open(output_json_path, "r") as file: + data = json.load(file) + return data + except json.JSONDecodeError as e: + if attempt_number == attempts - 1: + print(f"serve.py Failed to decode JSON after multiple attempts: {e}") + raise + else: + print(f"serve.py Attempt {attempt_number + 1} failed with JSONDecodeError, retrying...") + time.sleep(1) # Wait before retrying + except Exception as e: + print(f"serve.py Unexpected error reading file: {e}") + raise # Endpoint to read and serve JSON data from outputs.json @app.route("/miner-positions", methods=["GET"]) def get_miner_positions(): - api_key = get_api_key() + api_key = get_api_key() - # Check if the API key is valid - if api_key not in accessible_api_keys: - return jsonify({'error': 'Unauthorized access'}), 401 - else: - print("api key received.") + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 - f = "outputs/output.json" - data = get_file(f) + f = "outputs/output.json" + data = get_file(f) - if data is None: - return f"{f} not found", 404 - else: - return jsonify(data) + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) # serve miner positions v2 now named validator checkpoint @app.route("/validator-checkpoint", methods=["GET"]) def get_validator_checkpoint(): - api_key = get_api_key() + api_key = get_api_key() - # Check if the API key is valid - if api_key not in accessible_api_keys: - return jsonify({'error': 'Unauthorized access'}), 401 - else: - print("api key received.") + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 - f = "../runnable/validator_checkpoint.json" - data = get_file(f) + f = "../runnable/validator_checkpoint.json" + data = get_file(f) - if data is None: - return f"{f} not found", 404 - else: - return jsonify(data) + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) @app.route("/eliminations", methods=["GET"]) def get_eliminations(): - api_key = get_api_key() + api_key = get_api_key() - # Check if the API key is valid - if api_key not in accessible_api_keys: - return jsonify({'error': 'Unauthorized access'}), 401 - else: - print("api key received.") + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 - f = "eliminations.json" - data = get_file(f) + f = "eliminations.json" + data = get_file(f) - if data is None: - return f"{f} not found", 404 - else: - return jsonify(data) + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) @app.route("/miner-copying", methods=["GET"]) def get_miner_copying(): - api_key = get_api_key() + api_key = get_api_key() - # Check if the API key is valid - if api_key not in accessible_api_keys: - return jsonify({'error': 'Unauthorized access'}), 401 - else: - print("api key received.") + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 - f = "miner_copying.json" - data = get_file(f) + f = "miner_copying.json" + data = get_file(f) - if data is None: - return f"{f} not found", 404 - else: - return jsonify(data) + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) if __name__ == "__main__": - # sys.argv[0] is the script name itself - # Arguments start from sys.argv[1] - if len(sys.argv) > 1: - path = sys.argv[1] - else: - path = "../proprietary-trading-network/validation/" - print(path) - serve(app, host="127.0.0.1", port=48888) + # sys.argv[0] is the script name itself + # Arguments start from sys.argv[1] + if len(sys.argv) > 1: + path = sys.argv[1] + else: + path = "../proprietary-trading-network/validation/" + print(path) + serve(app, host="127.0.0.1", port=48888) From 84c9b459a015bcd07675a0f58ca52cb3a8c7549d Mon Sep 17 00:00:00 2001 From: jbonilla-tao <161871533+jbonilla-tao@users.noreply.github.com> Date: Fri, 17 May 2024 23:40:03 -0400 Subject: [PATCH 12/17] Create README.md --- README.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..8daa116 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ + +# Flask API Server for PTN Outputs + +## Overview +This Flask server is configured to serve the outputs of the [PTN repo (Bittensor subnet 8)](https://github.com/taoshidev/proprietary-trading-network/blob/main/docs/validator.md), which generates JSON files accessible via this API. The Flask code can easily be modified to serve outputs from any other Bittensor subnet. + +## Security Warning +The API uses a simple token-based authentication system. The default API key is set to "xxxx". **Change this default API key before deploying in a production environment to prevent unauthorized access.** + +## Configuration +### Changing the API Key +To enhance the security of your API, change the default API key in the `accessible_api_keys` list in `serve.py` to a more secure key. + +### Making Server Accessible +By default, the server binds to `127.0.0.1` which only allows local requests. To allow access from any IP address, bind to `0.0.0.0`: +```python +serve(app, host="0.0.0.0", port=48888) +``` + +## Security Considerations +### Rate Limiting +Consider implementing rate limiting using out of the box Flask extensions like `Flask-Limiter` or custom implementations to prevent abuse and ensure fair use of the API. + +### HTTPS +Deploy the Flask application over HTTPS in production to encrypt data in transit. This is typically done by placing the Flask application behind a reverse proxy that handles SSL/TLS termination. + +### Firewall Configuration +Configure firewall rules to only allow traffic on necessary ports from trusted IP addresses. + +## Usage with curl +Example `curl` commands to interact with the Flask server. Replace `` with your validator's IP address and `xxxx` with your API key that you hardcode in `serve.py` + +### Get Miner Positions with curl +```bash +curl -X GET http://:48888/miner-positions -H "Content-Type: application/json" -d '{"api_key": "xxxx"}' -o miner_positions.json +``` + +### Get Validator Checkpoint with curl +```bash +curl -X GET http://:48888/validator-checkpoint -H "Content-Type: application/json" -d '{"api_key": "xxxx"}' -o validator_checkpoint.json +``` + +## Usage with Python +Example python code to interact with the Flask server. + +### Get Validator Checkpoint with python + +```python +import requests +import json + +url = 'https://example.com/validator-checkpoint' +api_key = 'abcdefg' + +data = { +'api_key': api_key +} +json_data = json.dumps(data) +headers = { +'Content-Type': 'application/json', +} +test = requests.get(url, data=json_data, headers=headers) +print(test) +with open('validator_checkpoint.json', 'w') as f: + f.write(json.dumps(test.json())) +# print(json.loads(test.json())) + +``` + + +## Final Notes +This Flask server setup provides a simple template for serving API requests. If not using the Request Network, ensure all security measures are in place before deploying to a live environment. + +The Request Network is a Taoshi product which serves subnet data while handling security, rate limiting, data customization, and provide a polished customer-facing and validator setup UI. + From 208852d4f33cb7d0330201689297bd5205cd07c7 Mon Sep 17 00:00:00 2001 From: jbonilla-tao <161871533+jbonilla-tao@users.noreply.github.com> Date: Fri, 17 May 2024 23:46:13 -0400 Subject: [PATCH 13/17] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 8daa116..3e4f988 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ By default, the server binds to `127.0.0.1` which only allows local requests. To serve(app, host="0.0.0.0", port=48888) ``` +### Launching the server +On your validator, clone this repo, make desired edits, and then run +```bash +pm2 start serve.py --name serve +``` + ## Security Considerations ### Rate Limiting Consider implementing rate limiting using out of the box Flask extensions like `Flask-Limiter` or custom implementations to prevent abuse and ensure fair use of the API. From a9384c227d7d294e3b213755857d14d3b5697221 Mon Sep 17 00:00:00 2001 From: jbonilla-tao <161871533+jbonilla-tao@users.noreply.github.com> Date: Fri, 17 May 2024 23:49:04 -0400 Subject: [PATCH 14/17] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e4f988..5103598 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ serve(app, host="0.0.0.0", port=48888) ``` ### Launching the server -On your validator, clone this repo, make desired edits, and then run +On your validator, clone this repo in the same directory that the `proprietary-trading-network` repo is in, make desired edits, and then run ```bash pm2 start serve.py --name serve ``` From 8e11993eac2073890c8e4c52daf1a57b1ec5774f Mon Sep 17 00:00:00 2001 From: Thomas Dougherty Date: Mon, 20 May 2024 21:33:10 -0700 Subject: [PATCH 15/17] Adding miner statistics endpoint --- serve.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/serve.py b/serve.py index 4a284f5..533c411 100644 --- a/serve.py +++ b/serve.py @@ -62,7 +62,50 @@ def get_miner_positions(): return f"{f} not found", 404 else: return jsonify(data) + +@app.route("/miner-positions/", methods=["GET"]) +def get_miner_positions_unique(minerid): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "outputs/output.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + # Filter the data for the specified miner ID + filtered_data = data.get(minerid, None) + + if filtered_data is None: + return jsonify({'error': 'Miner ID not found'}), 404 + + return jsonify(filtered_data) + +# Endpoint to read and serve JSON data from outputs.json +@app.route("/miner-hotkeys", methods=["GET"]) +def get_miner_hotkeys(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "outputs/output.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + miner_hotkeys = list(data.keys()) + if len(miner_hotkeys) == 0: + return f"{f} not found", 404 + else: + return jsonify(miner_hotkeys) # serve miner positions v2 now named validator checkpoint @app.route("/validator-checkpoint", methods=["GET"]) @@ -80,6 +123,45 @@ def get_validator_checkpoint(): return f"{f} not found", 404 else: return jsonify(data) + +# serve miner positions v2 now named validator checkpoint +@app.route("/statistics", methods=["GET"]) +def get_validator_checkpoint_statistics(): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "../runnable/minerstatistics.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + else: + return jsonify(data) + +# serve miner positions v2 now named validator checkpoint +@app.route("/statistics//", methods=["GET"]) +def get_validator_checkpoint_statistics_unique(minerid): + api_key = get_api_key() + + # Check if the API key is valid + if api_key not in accessible_api_keys: + return jsonify({'error': 'Unauthorized access'}), 401 + + f = "../runnable/minerstatistics.json" + data = get_file(f) + + if data is None: + return f"{f} not found", 404 + + data_summary: list = data.get("data", None) + for element in data_summary: + if element.get("hotkey", None) == minerid: + return jsonify(element) + + return jsonify({'error': 'Miner ID not found'}), 404 @app.route("/eliminations", methods=["GET"]) From 6f94772f29f3ab2ffceb9aa3ac2170bf8b42143b Mon Sep 17 00:00:00 2001 From: wojtek-opentensor Date: Mon, 10 Jun 2024 16:42:15 +0000 Subject: [PATCH 16/17] Adding dockerfile. --- Dockerfile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dd02ce0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10-bookworm + +COPY . /app +WORKDIR /app + +EXPOSE 48888 +RUN pip install -r requirements.txt + +ENTRYPOINT ["python", "serve.py"] + From bf418ba3b08c0331a4965625f041c96f69b2fd24 Mon Sep 17 00:00:00 2001 From: Jordan Bonilla Date: Fri, 28 Jun 2024 22:28:32 -0400 Subject: [PATCH 17/17] Serve tiered position data --- serve.py | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/serve.py b/serve.py index 533c411..5296d93 100644 --- a/serve.py +++ b/serve.py @@ -1,6 +1,6 @@ import sys -from flask import Flask, jsonify, request +from flask import Flask, jsonify, request, Response import os import time import json @@ -25,15 +25,19 @@ def get_api_key(): return api_key -def get_file(f, attempts=3): - output_json_path = os.path.abspath(os.path.join(path, f)) - if not os.path.exists(output_json_path): +def get_file(f, attempts=3, binary=False): + file_path = os.path.abspath(os.path.join(path, f)) + if not os.path.exists(file_path): return None for attempt_number in range(attempts): try: - with open(output_json_path, "r") as file: - data = json.load(file) + if binary: + with open(file_path, 'rb') as f: + data = f.read() + else: + with open(file_path, "r") as file: + data = json.load(file) return data except json.JSONDecodeError as e: if attempt_number == attempts - 1: @@ -55,13 +59,32 @@ def get_miner_positions(): if api_key not in accessible_api_keys: return jsonify({'error': 'Unauthorized access'}), 401 - f = "outputs/output.json" - data = get_file(f) + # Get the 'tier' query parameter from the request + tier = request.args.get('tier') + is_gz_data = tier is not None + + if is_gz_data: + # Validate the 'tier' parameter + if tier not in ['0', '30', '50', '100']: + return jsonify({'error': 'Invalid tier value. Allowed values are 0, 30, 50, or 100'}), 400 + + # Construct the relative path based on the specified tier + f = f"outputs/tiered_positions/{tier}/output.json.gz" + else: + # If 'tier' parameter is not provided, return the default output.json + f = "outputs/output.json" + + # Attempt to retrieve the file + data = get_file(f, binary=is_gz_data) if data is None: return f"{f} not found", 404 - else: - return jsonify(data) + if is_gz_data: + return Response(data, content_type='application/json', headers={ + 'Content-Encoding': 'gzip' + }) + return jsonify(data) + @app.route("/miner-positions/", methods=["GET"]) def get_miner_positions_unique(minerid):