diff --git a/src/live_data_server/plots/models.py b/src/live_data_server/plots/models.py index 81f5e43..8a53c8c 100644 --- a/src/live_data_server/plots/models.py +++ b/src/live_data_server/plots/models.py @@ -19,7 +19,7 @@ class Instrument(models.Model): name = models.CharField(max_length=128, unique=True) run_id_type = models.IntegerField(default=0) - def __unicode__(self): + def __str__(self): return self.name @@ -35,8 +35,8 @@ class DataRun(models.Model): instrument = models.ForeignKey(Instrument, on_delete=models.deletion.CASCADE) created_on = models.DateTimeField("Timestamp", auto_now_add=True) - def __unicode__(self): - return "%s_%d_%s" % (self.instrument, self.run_number, self.run_id) + def __str__(self): + return f"{self.instrument}_{self.run_number}_{self.run_id}" class PlotData(models.Model): @@ -56,14 +56,8 @@ class PlotData(models.Model): timestamp = models.DateTimeField("Timestamp") - def __unicode__(self): - return "%s" % self.data_run - - def is_div(self): - """ - Return whether the data is a
- """ - return self.data_type % 100 == 1 + def __str__(self): + return str(self.data_run) def is_data_type_valid(self, data_type): """ @@ -93,19 +87,3 @@ def get_data_type_from_string(cls, type_string): Returns the correct data type ID for a given string representation """ return DATA_TYPES.get(type_string, DATA_TYPES["json"]) - - @classmethod - def data_type_as_string(cls, data_type): - """ - Return an internal name to use for a given data_type. - This name is generally used in function names and relates - to the data format (json or html). In principle, different - data types can return the same string. - - @param data_type: data type ID [integer] - """ - data_type = int(data_type) - data_type_info = DATA_TYPE_INFO.get(data_type) - if data_type_info is not None: - return data_type_info["name"] - return None diff --git a/src/live_data_server/plots/urls.py b/src/live_data_server/plots/urls.py index e0d859b..cf56ae7 100644 --- a/src/live_data_server/plots/urls.py +++ b/src/live_data_server/plots/urls.py @@ -10,7 +10,6 @@ app_name = "plots" urlpatterns = [ - re_path(r"^(?P[\w]+)/(?P\d+)/$", views.live_plot, name="live_plot"), re_path(r"^(?P[\w]+)/(?P\d+)/update/json/$", views.update_as_json, name="update_as_json"), re_path(r"^(?P[\w]+)/(?P\d+)/update/html/$", views.update_as_html, name="update_as_html"), re_path( diff --git a/src/live_data_server/plots/views.py b/src/live_data_server/plots/views.py index 9ccfdcc..6bd12a6 100644 --- a/src/live_data_server/plots/views.py +++ b/src/live_data_server/plots/views.py @@ -8,10 +8,8 @@ from django.conf import settings from django.contrib.auth import authenticate, login -from django.core.exceptions import PermissionDenied from django.http import HttpResponse, HttpResponseNotFound, JsonResponse -from django.shortcuts import get_object_or_404, render_to_response -from django.urls import reverse +from django.shortcuts import get_object_or_404 from django.utils import dateformat, timezone from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt @@ -21,15 +19,6 @@ from . import view_util -def _check_credentials(request): - """ - Internal utility method to check whether a user has access to a view - """ - # If we don't allow guests but the user is authenticated, return the function - if request.user.is_authenticated: - return True - - def check_credentials(fn): """ Function decorator to authenticate a request @@ -48,33 +37,14 @@ def request_processor(request, *args, **kws): if request_user is not None and not request_user.is_anonymous: login(request, request_user) return fn(request, *args, **kws) + else: + return HttpResponse(status=401) else: - raise PermissionDenied + return HttpResponse(status=401) return request_processor -def live_plot(request, instrument, run_id): - """ - Test view for live plotting. - @param instrument: instrument name - @param run_id: run number - """ - data_type_default = PlotData.get_data_type_from_string("html") - data_type = request.GET.get("data_type", default=data_type_default) - update_url = reverse( - "plots:update_as_%s" % PlotData.data_type_as_string(data_type), - kwargs={"instrument": instrument, "run_id": run_id}, - ) - client_key = view_util.generate_key(instrument, run_id) - if client_key is not None: - update_url += "?key=%s" % client_key - template_values = {} - template_values["data_type"] = data_type - template_values["update_url"] = update_url - return render_to_response("plots/live_plot.html", template_values) - - @view_util.check_key @cache_page(15) def update_as_json(request, instrument, run_id): # noqa: ARG001 @@ -87,7 +57,7 @@ def update_as_json(request, instrument, run_id): # noqa: ARG001 plot_data = view_util.get_plot_data(instrument, run_id, data_type=data_type) if plot_data is None: - error_msg = "No data available for %s %s" % (instrument, run_id) + error_msg = f"No data available for {instrument} {run_id}" logging.error(error_msg) return HttpResponseNotFound(error_msg) @@ -107,7 +77,7 @@ def update_as_html(request, instrument, run_id): # noqa: ARG001 plot_data = view_util.get_plot_data(instrument, run_id, data_type=data_type) if plot_data is None: - error_msg = "No data available for %s %s" % (instrument, run_id) + error_msg = f"No data available for {instrument} {run_id}" logging.error(error_msg) return HttpResponseNotFound(error_msg) @@ -124,7 +94,7 @@ def _store(request, instrument, run_id=None, as_user=False): @param run_id: run number @param as_user: if True, we will store as user data """ - if request.user.is_authenticated and "file" in request.FILES: + if "file" in request.FILES: raw_data = request.FILES["file"].read().decode("utf-8") data_type_default = PlotData.get_data_type_from_data(raw_data) data_type = request.POST.get("data_type", default=data_type_default) @@ -134,7 +104,7 @@ def _store(request, instrument, run_id=None, as_user=False): else: view_util.store_plot_data(instrument, run_id, raw_data, data_type) else: - raise PermissionDenied + return HttpResponse(status=400) return HttpResponse() @@ -160,26 +130,22 @@ def upload_user_data(request, user): @csrf_exempt @check_credentials -def get_data_list(request, instrument): +def get_data_list(_, instrument): """ Get a list of user data """ - if request.user.is_authenticated: - instrument_object = get_object_or_404(Instrument, name=instrument.lower()) - data_list = [] - for item in DataRun.objects.filter(instrument=instrument_object): - localtime = timezone.localtime(item.created_on) - df = dateformat.DateFormat(localtime) - data_list.append( - dict( - id=item.id, - run_number=str(item.run_number), - run_id=item.run_id, - timestamp=item.created_on.isoformat(), - created_on=df.format(settings.DATETIME_FORMAT), - ) + instrument_object = get_object_or_404(Instrument, name=instrument.lower()) + data_list = [] + for item in DataRun.objects.filter(instrument=instrument_object): + localtime = timezone.localtime(item.created_on) + df = dateformat.DateFormat(localtime) + data_list.append( + dict( + id=item.id, + run_number=str(item.run_number), + run_id=item.run_id, + timestamp=item.created_on.isoformat(), + created_on=df.format(settings.DATETIME_FORMAT), ) - response = HttpResponse(json.dumps(data_list), content_type="application/json") - return response - else: - raise PermissionDenied + ) + return JsonResponse(data_list, safe=False) diff --git a/src/live_data_server/templates/base.html b/src/live_data_server/templates/base.html deleted file mode 100644 index 6936315..0000000 --- a/src/live_data_server/templates/base.html +++ /dev/null @@ -1,26 +0,0 @@ -{% load staticfiles %} - - - - {% block head %} - plotly example - - - - - {% endblock %} - - - {% block content %} - - - {% endblock %} - - {% block tail %} - - - - - {% endblock %} - - diff --git a/src/live_data_server/templates/plots/live_plot.html b/src/live_data_server/templates/plots/live_plot.html deleted file mode 100644 index 1e3eaba..0000000 --- a/src/live_data_server/templates/plots/live_plot.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base.html" %} -{% block head %} -{{block.super}} - -{% endblock %} - -{% block content %} - -

Live plot

-

Updates every few seconds

-
- - -{% endblock %} diff --git a/tests/test_post_get.py b/tests/test_post_get.py index c8d9b3c..3104e01 100644 --- a/tests/test_post_get.py +++ b/tests/test_post_get.py @@ -3,88 +3,228 @@ import json import os +import psycopg2 import requests - -def test_post_request(data_server): - http_ok = requests.status_codes.codes["OK"] - username = os.environ.get("DJANGO_SUPERUSER_USERNAME") - monitor_user = {"username": username, "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD")} - print(monitor_user) - # load html plot as autoreduce service - file_name = "reflectivity.html" - files = {"file": open(data_server.path_to(file_name)).read()} - monitor_user["data_id"] = file_name - - http_request = requests.post( - "http://127.0.0.1:80/plots/REF_L/12345/upload_plot_data/", data=monitor_user, files=files, verify=True - ) - assert http_request.status_code == http_ok - - # load json plot a user "someuser" of the web-reflectivity app - file_name = "reflectivity.json" - with open(data_server.path_to(file_name), "r") as file_handle: - files = {"file": json.dumps(json.load(file_handle))} - monitor_user["data_id"] = file_name - - http_request = requests.post( - "http://127.0.0.1:80/plots/" + username + "/upload_user_data/", data=monitor_user, files=files, verify=True - ) - assert http_request.status_code == http_ok - - monitor_user.pop("data_id") - # get all plots for an instrument - http_request = requests.post("http://127.0.0.1:80/plots/REF_L/list/", data=monitor_user, files={}, verify=True) - assert http_request.status_code == http_ok - - # get all plots from someuser - http_request = requests.post( - "http://127.0.0.1:80/plots/" + username + "/list/", data=monitor_user, files={}, verify=True - ) - assert http_request.status_code == http_ok - - -def test_get_request(data_server): - """Test GET request for HTML data like from monitor.sns.gov""" - instrument = "REF_M" - run_number = 12346 - http_ok = requests.status_codes.codes["OK"] - http_unauthorized = requests.status_codes.codes["unauthorized"] - - # upload the run data using POST (authenticate with username and password) - username = os.environ.get("DJANGO_SUPERUSER_USERNAME") - monitor_user = {"username": username, "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD")} - # load html plot as autoreduce service - file_name = "reflectivity.html" - files = {"file": open(data_server.path_to(file_name)).read()} - monitor_user["data_id"] = file_name - - http_request = requests.post( - f"http://127.0.0.1:80/plots/{instrument}/{run_number}/upload_plot_data/", - data=monitor_user, - files=files, - verify=True, - ) - assert http_request.status_code == http_ok - - base_url = f"http://127.0.0.1:80/plots/{instrument}/{run_number}/update/html/" - - # test GET request - authenticate with secret key - url = f"{base_url}?key={_generate_key(instrument, run_number)}" - http_request = requests.get(url) - assert http_request.status_code == http_ok - assert http_request.text == files["file"] - - # test GET request - no key - # TODO: this should return 401 unauthorized - url = base_url - http_request = requests.get(url) - assert http_request.status_code == http_ok - - # test GET request - wrong key - url = f"{base_url}?key=WRONG-KEY" - http_request = requests.get(url) - assert http_request.status_code == http_unauthorized +TEST_URL = "http://127.0.0.1" +HTTP_OK = requests.status_codes.codes["OK"] +HTTP_UNAUTHORIZED = requests.status_codes.codes["unauthorized"] +HTTP_NOT_FOUND = requests.status_codes.codes["NOT_FOUND"] +HTTP_BAD_REQUEST = requests.status_codes.codes["BAD_REQUEST"] + + +class TestLiveDataServer: + @classmethod + def setup_class(cls): + """Clean the database before running tests""" + conn = psycopg2.connect( + database=os.environ.get("DATABASE_NAME"), + user=os.environ.get("DATABASE_USER"), + password=os.environ.get("DATABASE_PASS"), + port=os.environ.get("DATABASE_PORT"), + host="localhost", + ) + cur = conn.cursor() + cur.execute("DELETE FROM plots_plotdata") + cur.execute("DELETE FROM plots_datarun") + cur.execute("DELETE FROM plots_instrument") + conn.commit() + conn.close() + + def test_post_request(self, data_server): + username = os.environ.get("DJANGO_SUPERUSER_USERNAME") + monitor_user = {"username": username, "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD")} + + # load html plot as autoreduce service + file_name = "reflectivity.html" + files = {"file": open(data_server.path_to(file_name)).read()} + monitor_user["data_id"] = file_name + + http_request = requests.post( + TEST_URL + "/plots/REF_L/12345/upload_plot_data/", data=monitor_user, files=files, verify=True + ) + assert http_request.status_code == HTTP_OK + + # load json plot a user "someuser" of the web-reflectivity app + file_name = "reflectivity.json" + with open(data_server.path_to(file_name), "r") as file_handle: + files = {"file": json.dumps(json.load(file_handle))} + monitor_user["data_id"] = file_name + + http_request = requests.post( + TEST_URL + "/plots/" + username + "/upload_user_data/", data=monitor_user, files=files, verify=True + ) + assert http_request.status_code == HTTP_OK + + monitor_user.pop("data_id") + # get all plots for an instrument + http_request = requests.post(TEST_URL + "/plots/REF_L/list/", data=monitor_user, files={}, verify=True) + assert http_request.status_code == HTTP_OK + + # get all plots from someuser + http_request = requests.post( + TEST_URL + "/plots/" + username + "/list/", data=monitor_user, files={}, verify=True + ) + assert http_request.status_code == HTTP_OK + + def test_get_request(self, data_server): + """Test GET request for HTML data like from monitor.sns.gov""" + instrument = "REF_M" + run_number = 12346 + + # upload the run data using POST (authenticate with username and password) + username = os.environ.get("DJANGO_SUPERUSER_USERNAME") + monitor_user = {"username": username, "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD")} + # load html plot as autoreduce service + file_name = "reflectivity.html" + files = {"file": open(data_server.path_to(file_name)).read()} + monitor_user["data_id"] = file_name + + http_request = requests.post( + f"{TEST_URL}/plots/{instrument}/{run_number}/upload_plot_data/", + data=monitor_user, + files=files, + verify=True, + ) + assert http_request.status_code == HTTP_OK + + base_url = f"{TEST_URL}/plots/{instrument}/{run_number}/update/html/" + + # test GET request - authenticate with secret key + url = f"{base_url}?key={_generate_key(instrument, run_number)}" + http_request = requests.get(url) + assert http_request.status_code == HTTP_OK + assert http_request.text == files["file"] + + # test that getting the json should return not found + http_request = requests.get( + f"{TEST_URL}/plots/{instrument}/{run_number}/update/json/?key={_generate_key(instrument, run_number)}" + ) + assert http_request.status_code == HTTP_NOT_FOUND + assert http_request.text == "No data available for REF_M 12346" + + # test GET request - no key + # TODO: this should return 401 unauthorized + url = base_url + http_request = requests.get(url) + assert http_request.status_code == HTTP_OK + + # test GET request - wrong key + url = f"{base_url}?key=WRONG-KEY" + http_request = requests.get(url) + assert http_request.status_code == HTTP_UNAUTHORIZED + + def test_upload_plot_data_json(self): + # test that when you upload json you can get back the same stuff + instrument = "instrument0" + run_number = 123 + data = {"a": "A", "b": 1, "c": ["C", 2]} + + monitor_user = { + "username": os.environ.get("DJANGO_SUPERUSER_USERNAME"), + "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD"), + } + + # get that there is currently no data + response = requests.post(f"{TEST_URL}/plots/{instrument}/list/", data=monitor_user) + assert response.status_code == HTTP_NOT_FOUND + + # now upload json data + http_request = requests.post( + f"{TEST_URL}/plots/{instrument}/{run_number}/upload_plot_data/", + data=monitor_user, + files={"file": json.dumps(data)}, + ) + assert http_request.status_code == HTTP_OK + + # check list of data + response = requests.post(f"{TEST_URL}/plots/{instrument}/list/", data=monitor_user) + assert response.status_code == HTTP_OK + data_list = response.json() + assert len(data_list) == 1 + assert data_list[0]["run_number"] == "123" + + # now get the data as json + response = requests.get( + f"{TEST_URL}/plots/{instrument}/{run_number}/update/json/?key={_generate_key(instrument, run_number)}" + ) + assert response.status_code == HTTP_OK + assert response.headers["Content-Type"] == "application/json" + assert response.json() == data + + # now try getting it as html, should fail + response = requests.get( + f"{TEST_URL}/plots/{instrument}/{run_number}/update/html/?key={_generate_key(instrument, run_number)}" + ) + assert response.status_code == HTTP_NOT_FOUND + assert response.text == "No data available for instrument0 123" + + def test_bad_request(self): + instrument = "instrument1" + run_number = 1234 + + # test if missing files in post request + monitor_user = { + "username": os.environ.get("DJANGO_SUPERUSER_USERNAME"), + "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD"), + } + + # missing files + http_request = requests.post( + f"{TEST_URL}/plots/{instrument}/{run_number}/upload_plot_data/", + data=monitor_user, + ) + assert http_request.status_code == HTTP_BAD_REQUEST + + # used filename instead of file in files + http_request = requests.post( + f"{TEST_URL}/plots/{instrument}/{run_number}/upload_plot_data/", + data=monitor_user, + files={"filename": ""}, + ) + assert http_request.status_code == HTTP_BAD_REQUEST + + def test_unauthorized(self): + # test get request unauthorized + monitor_user = { + "username": os.environ.get("DJANGO_SUPERUSER_USERNAME"), + "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD"), + } + response = requests.get(f"{TEST_URL}/plots/instrument/list/", data=monitor_user) + assert response.status_code == HTTP_UNAUTHORIZED + + # test wrong password + monitor_user = { + "username": os.environ.get("DJANGO_SUPERUSER_USERNAME"), + "password": "WrongPassword", + } + response = requests.post(f"{TEST_URL}/plots/instrument/list/", data=monitor_user) + assert response.status_code == HTTP_UNAUTHORIZED + + # missing username and password + response = requests.post(f"{TEST_URL}/plots/instrument/list/") + assert response.status_code == HTTP_UNAUTHORIZED + + def test_session(self): + # once you authenicate once with username and password you should be able to reuse the session with credentials + session = requests.Session() + + monitor_user = { + "username": os.environ.get("DJANGO_SUPERUSER_USERNAME"), + "password": os.environ.get("DJANGO_SUPERUSER_PASSWORD"), + } + + # initial get should be unauthorized + response = session.get(f"{TEST_URL}/plots/not_a_instrument/list/") + assert response.status_code == HTTP_UNAUTHORIZED + + # do post with username and password, expect not found for "not_a_instrument" + response = session.post(f"{TEST_URL}/plots/not_a_instrument/list/", data=monitor_user) + response.status_code == HTTP_NOT_FOUND + + # now get with same session should be authorized + response = session.get(f"{TEST_URL}/plots/not_a_instrument/list/") + response.status_code == HTTP_NOT_FOUND def _generate_key(instrument, run_id):