diff --git a/analytics_automated/api.py b/analytics_automated/api.py index 88dabd25..2ccd101b 100644 --- a/analytics_automated/api.py +++ b/analytics_automated/api.py @@ -1,6 +1,6 @@ import ast import uuid -from ipware.ip import get_ip +from ipware.ip import get_client_ip from collections import defaultdict import pprint import logging @@ -181,7 +181,7 @@ def __prepare_data(self, request): data['submission_name'] = request_contents.pop('submission_name') data['email'] = request_contents.pop('email') data['job'] = request_contents.pop('job') - data['ip'] = get_ip(request) + data['ip'] = get_client_ip(request)[0] # data['UUID'] = str(uuid.uuid1()) except MultiValueDictKeyError: raise MultiValueDictKeyError diff --git a/analytics_automated/tests/model_factories.py b/analytics_automated/tests/model_factories.py index 0df94c10..8011e1d9 100644 --- a/analytics_automated/tests/model_factories.py +++ b/analytics_automated/tests/model_factories.py @@ -27,7 +27,7 @@ def random_string(length=10): return u''.join(random.choice(string.ascii_letters) for x in range(length)) -class QueueTypeFactory(factory.DjangoModelFactory): +class QueueTypeFactory(factory.django.DjangoModelFactory): name = "localhost" execution_behaviour = QueueType.LOCALHOST @@ -36,7 +36,7 @@ class Meta: django_get_or_create = ('name',) -class BackendFactory(factory.DjangoModelFactory): +class BackendFactory(factory.django.DjangoModelFactory): name = factory.Sequence(lambda n: 'test_{}'.format(n)) queue_type = factory.SubFactory(QueueTypeFactory) # ip = ".".join(map(str, (random.randint(0, 255) for _ in range(4)))) @@ -48,7 +48,7 @@ class Meta: django_get_or_create = ('name',) -class TaskFactory(factory.DjangoModelFactory): +class TaskFactory(factory.django.DjangoModelFactory): backend = factory.SubFactory(BackendFactory) name = factory.Sequence(lambda n: 'task_{}'.format(n)) in_glob = factory.LazyAttribute(lambda t: random_string()) @@ -63,7 +63,7 @@ class Meta: django_get_or_create = ('name',) -class ConfigurationFactory(factory.DjangoModelFactory): +class ConfigurationFactory(factory.django.DjangoModelFactory): task = factory.SubFactory(TaskFactory) type = random.randint(0, 2) name = factory.LazyAttribute(lambda t: random_string()) @@ -74,7 +74,7 @@ class Meta: model = Configuration -class JobFactory(factory.DjangoModelFactory): +class JobFactory(factory.django.DjangoModelFactory): name = factory.Sequence(lambda n: 'job_{}'.format(n)) runnable = True @@ -83,7 +83,7 @@ class Meta: django_get_or_create = ('name',) -class StepFactory(factory.DjangoModelFactory): +class StepFactory(factory.django.DjangoModelFactory): job = factory.SubFactory(JobFactory) task = factory.SubFactory(TaskFactory) ordering = random.randint(0, 20) @@ -92,7 +92,7 @@ class Meta: model = Step -class BatchFactory(factory.DjangoModelFactory): +class BatchFactory(factory.django.DjangoModelFactory): UUID = factory.LazyAttribute(lambda t: str(uuid.uuid1())) status = random.randint(0, 4) @@ -100,7 +100,7 @@ class Meta: model = Batch -class SubmissionFactory(factory.DjangoModelFactory): +class SubmissionFactory(factory.django.DjangoModelFactory): job = factory.SubFactory(JobFactory) submission_name = factory.Sequence(lambda n: 'submission_{}'.format(n)) UUID = factory.LazyAttribute(lambda t: str(uuid.uuid1())) @@ -115,7 +115,7 @@ class Meta: django_get_or_create = ('submission_name',) -class ParameterFactory(factory.DjangoModelFactory): +class ParameterFactory(factory.django.DjangoModelFactory): task = factory.SubFactory(TaskFactory) flag = factory.LazyAttribute(lambda t: random_string()) default = factory.LazyAttribute(lambda t: random_string()) @@ -126,7 +126,7 @@ class Meta: model = Parameter -class EnvironmentFactory(factory.DjangoModelFactory): +class EnvironmentFactory(factory.django.DjangoModelFactory): task = factory.SubFactory(TaskFactory) env = factory.LazyAttribute(lambda t: random_string()) value = factory.LazyAttribute(lambda t: random_string()) @@ -135,14 +135,14 @@ class Meta: model = Environment -class ValidatorTypesFactory(factory.DjangoModelFactory): +class ValidatorTypesFactory(factory.django.DjangoModelFactory): name = factory.LazyAttribute(lambda t: random_string()) class Meta: model = ValidatorTypes -class ValidatorFactory(factory.DjangoModelFactory): +class ValidatorFactory(factory.django.DjangoModelFactory): job = factory.SubFactory(JobFactory) validation_type = factory.SubFactory(ValidatorTypesFactory) @@ -150,7 +150,7 @@ class Meta: model = Validator -class ResultFactory(factory.DjangoModelFactory): +class ResultFactory(factory.django.DjangoModelFactory): submission = factory.SubFactory(SubmissionFactory) task = factory.SubFactory(TaskFactory) step = step_value diff --git a/analytics_automated_project/settings/base.py b/analytics_automated_project/settings/base.py index 8f29c709..2aaf4154 100644 --- a/analytics_automated_project/settings/base.py +++ b/analytics_automated_project/settings/base.py @@ -136,7 +136,7 @@ def get_secret(setting, secrets): # Application definition INSTALLED_APPS = ( - 'django_admin_bootstrapped', + 'bootstrap_admin', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -306,3 +306,5 @@ def get_secret(setting, secrets): } } } + +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/analytics_automated_project/settings/dev.py b/analytics_automated_project/settings/dev.py index 0fb7314e..9c873ebe 100644 --- a/analytics_automated_project/settings/dev.py +++ b/analytics_automated_project/settings/dev.py @@ -20,8 +20,8 @@ } CORS_ORIGIN_WHITELIST = ( - '127.0.0.1:4000', - '127.0.0.1:8000', + 'http://127.0.0.1:4000', + 'http://127.0.0.1:8000', ) SECRET_KEY = get_secret("SECRET_KEY", secrets) diff --git a/example_scripts/send_fasta.py b/example_scripts/send_fasta.py index 70357d49..abe20f25 100644 --- a/example_scripts/send_fasta.py +++ b/example_scripts/send_fasta.py @@ -2,10 +2,11 @@ import requests import json -url = 'http://bioinf.cs.ucl.ac.uk/psipred_beta/api/submission.json' +# url = 'http://bioinf.cs.ucl.ac.uk/psipred_beta/api/submission.json' +url = 'http://127.0.0.1:8000/analytics_automated/submission.json' -payload = {'input_data': ('prot.txt', open('../submissions/files/prot.txt', 'rb'))} -data = {'job': 'disopred', +payload = {'input_data': ('prot.txt', open('./submissions/files/prot.txt', 'rb'))} +data = {'job': 'psipred', 'submission_name': 'testing', 'email': 'daniel.buchan@ucl.ac.uk', } r = requests.post(url, data=data, files=payload) diff --git a/requirements/base.txt b/requirements/base.txt index 9ad5d8d6..c311d490 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,64 +1,94 @@ -alabaster==0.7.6 -amqp==2.4.1 +alabaster==0.7.12 +amqp==2.6.1 anyjson==0.3.3 -Babel==2.3.4 -billiard==3.5.0.2 -bugsnag==2.3.1 -celery==4.2.1 -certifi==2019.3.9 -chardet==3.0.4 +asgiref==3.4.1 +attrs==21.2.0 +Babel==2.9.1 +billiard==3.6.4.0 +bootstrap-admin==0.4.4 +bugsnag==4.1.0 +celery==4.4.7 +certifi==2021.5.30 +cffi==1.14.6 +chardet==4.0.0 +charset-normalizer==2.0.3 commandRunner==0.8.6 -coverage==3.7.1 -defusedxml==0.4.1 -Django==2.2 +coverage==5.5 +defusedxml==0.7.1 +Django==3.2.5 django-admin-bootstrapped==2.5.6 -django-bootstrap3==11.0.0 -django-celery-beat==1.4.0 -django-cors-headers==2.5.2 -django-debug-toolbar==1.10.0 -django-filter==1.0.4 -django-guardian==1.3 -django-ipware==1.0.0 -django-registration-redux==1.2 -django-smuggler==0.7.0 -django-timezone-field==3.0 -djangorestframework==3.9.2 -djangorestframework-xml==1.4.0 +django-bootstrap3==15.0.0 +django-celery-beat==1.6.0 +django-cors-headers==3.7.0 +django-debug-toolbar==3.2.1 +django-filter==2.4.0 +django-guardian==2.4.0 +django-hstore==1.4.2 +django-ipware==3.0.2 +django-registration-redux==2.9 +django-smuggler==1.0.2 +django-timezone-field==3.1 +django-utils-six==2.0 +djangorestframework==3.12.4 +djangorestframework-hstore==1.3 +djangorestframework-xml==2.0.0 DjangoRestless==0.0.10 -docutils==0.12 -dr-dump==0.2.5 -drmaa==0.7.6 -factory-boy==2.8.1 -Faker==0.7.7 -flower==0.9.1 -idna==2.8 -Jinja2==2.7.3 -kombu==4.3.0 -MarkupSafe==0.23 -mod-wsgi==4.6.5 -mod-wsgi-httpd==2.4.35.1 -numpy==1.10.1 -Pillow==2.8.1 -psycopg2==2.7.3.2 -pyaml==18.11.0 -Pygments==2.0.2 -pyRserve==0.8.1 -python-crontab==2.3.6 -python-dateutil==2.6.0 -pytz==2017.2 -PyYAML==3.12 -redis==2.10.5 -requests==2.21.0 -rpy2==2.8.5 -scipy==0.19.0 -six==1.10.0 -snowballstemmer==1.2.0 -Sphinx==1.3.1 -sphinx-rtd-theme==0.1.8 -sqlparse==0.2.2 -tornado==4.2 +docutils==0.16 +dr-dump==1.1.4 +drmaa==0.7.9 +factory-boy==3.2.0 +Faker==8.10.3 +flower==0.9.7 +humanize==3.10.0 +idna==3.2 +imagesize==1.2.0 +importlib-metadata==2.1.1 +iniconfig==1.1.1 +Jinja2==3.0.1 +kombu==4.6.11 +MarkupSafe==2.0.1 +mod-wsgi==4.8.0 +mod-wsgi-httpd==2.4.46.1 +numpy==1.19.5 +packaging==21.0 +Pillow==8.3.1 +pluggy==0.13.1 +prometheus-client==0.8.0 +psycopg2==2.9.1 +py==1.10.0 +pyaml==20.4.0 +pycparser==2.20 +Pygments==2.9.0 +pyparsing==2.4.7 +pyRserve==0.9.2 +pytest==6.2.4 +python-crontab==2.5.1 +python-dateutil==2.8.2 +pytz==2021.1 +PyYAML==5.4.1 +redis==3.5.3 +requests==2.26.0 +rpy2==3.4.5 +scipy==1.5.4 +six==1.16.0 +snowballstemmer==2.1.0 +Sphinx==4.1.2 +sphinx-rtd-theme==0.5.2 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +sqlparse==0.4.1 +text-unidecode==1.3 +toml==0.10.2 +tornado==6.1 +typing-extensions==3.10.0.0 +tzlocal==2.1 Unipath==1.1 -urllib3==1.24.1 -vine==1.1.3 -WebOb==1.5.0 -xmltodict==0.9.2 +urllib3==1.26.6 +vine==1.3.0 +WebOb==1.8.7 +xmltodict==0.12.0 +zipp==3.5.0 diff --git a/utility_scripts/aa.hist b/utility_scripts/aa.hist index 8a86fad1..cbfd0790 100644 --- a/utility_scripts/aa.hist +++ b/utility_scripts/aa.hist @@ -1,3 +1,6 @@ python manage.py runserver --settings=analytics_automated_project.settings.dev celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python workon analytics_automated +exit +celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast_backend,high_blast_backend,low_blast_backend +celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast,high_blast,low_blast diff --git a/utility_scripts/spawn_dev_processes.sh b/utility_scripts/spawn_dev_processes.sh new file mode 100644 index 00000000..c45ee0bf --- /dev/null +++ b/utility_scripts/spawn_dev_processes.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +#possibly more the postmaster.pid if the machine rebooted +gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'postmaster -D /home/dbuchan/Databases/postgres_96_db/;' +gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'sleep 2; redis-server;' +gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash --rcfile ~/.bashrc -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; python manage.py runserver --settings=analytics_automated_project.settings.dev; exec bash"' +gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python; exec bash"' diff --git a/utility_scripts/spawn_dev_processes_dbuchan.sh b/utility_scripts/spawn_dev_processes_dbuchan.sh index c45ee0bf..6dbd771b 100755 --- a/utility_scripts/spawn_dev_processes_dbuchan.sh +++ b/utility_scripts/spawn_dev_processes_dbuchan.sh @@ -4,4 +4,4 @@ gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'postmaster -D /home/dbuchan/Databases/postgres_96_db/;' gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'sleep 2; redis-server;' gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash --rcfile ~/.bashrc -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; python manage.py runserver --settings=analytics_automated_project.settings.dev; exec bash"' -gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python; exec bash"' +gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast,high_blast,low_blast; exec bash"'