diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bad25e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,190 @@ +# Created by https://www.toptal.com/developers/gitignore/api/linux,django +# Edit at https://www.toptal.com/developers/gitignore?templates=linux,django + +data/ +### Django ### +*.log +*.pot +*.pyc +__pycache__/ +local_settings.py +db.sqlite3 +db.sqlite3-journal +media + +# If your build process includes running collectstatic, then you probably don't need or want to include staticfiles/ +# in your Git repository. Update and uncomment the following line accordingly. +# /staticfiles/ + +### Django.Python Stack ### +# Byte-compiled / optimized / DLL files +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo + +# Django stuff: + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +# End of https://www.toptal.com/developers/gitignore/api/linux,django \ No newline at end of file diff --git a/quiz/01-rate-limit/quiz01/Dockerfile b/quiz/01-rate-limit/quiz01/Dockerfile new file mode 100644 index 0000000..464cf2a --- /dev/null +++ b/quiz/01-rate-limit/quiz01/Dockerfile @@ -0,0 +1,9 @@ +# syntax=docker/dockerfile:1 +FROM python:3.9 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /code +COPY requirements.txt /code/ +COPY locustfile.py /code/ +RUN pip install -r requirements.txt +COPY . /code/ diff --git a/quiz/01-rate-limit/quiz01/README.md b/quiz/01-rate-limit/quiz01/README.md new file mode 100644 index 0000000..627f1b5 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/README.md @@ -0,0 +1,71 @@ +# 01-rate-limit +>Rate limiting is used to protect resources from being over-using or abused by users/bots/applications. It is commonly implemented by social media platforms such as Facebook or Instagram. +## The project components +用Django做應用框架搭配 [django-ratelimit](https://django-ratelimit.readthedocs.io/en/stable/)做速率限制，在資料庫上用Sqlite做每個使用者的header file的紀錄。 + + +## Usage +1. 建立Image，tag加入"v3.3.1"，或者自行至`docker-compose.yml`修改web的image的賦予名稱 + +``` +docker build -t="v3.3.1" . +``` +2. 載入[locustio/locust](https://hub.docker.com/r/locustio/locust)，Load testing 的工具 +``` +docker pull locustio/locust +``` + +3. Migration(資料遷移) +``` +docker-compose run web python3 manage.py makemigrations +``` +``` +docker-compose run web python3 manage.py migrate +``` + +4. 啟動container +``` +docker-compose up --scale worker=4 +``` +5. 開啟網頁，導向 http://localhost:8000/api/ + +![image](/quiz/01-rate-limit/quiz01/image/webpage.png) + + +## Test +使用Postman和撰寫Django Test做基本ratelimit測試，也加入[Locust](https://locust.io/)做Load testing 和 Profiling 評估。 +### Postman +Postman 是常用的api測試工具，我們可以透過Postman簡單的進行Request，也可以透過該工具清楚的看到response的header field。 + +**Get** +![image](/quiz/01-rate-limit/quiz01//image/postman_get.png "This is a sample image.") + +**Post** +![image](/quiz/01-rate-limit/quiz01//image/postman_post.png "This is a sample image.") + +**429 Too Many Requests** +![image](/quiz/01-rate-limit/quiz01//image/postman_429.png "This is a sample image.") + +### Django Test +透過Django Test 可以自訂義test的方法，查看response header內容，並顯示執行時間，如果超過呼叫次數顯示"Over Rating"。 +``` +docker-compose run web python3 manage.py test +``` +* 自訂義Django Test 4種測試: + 1. 在1秒內Request Get 100個請求(Get Not Over) + 2. 在1秒內Request Post 1個請求(Post Not Over) + 3. 在1秒內Request Get 101個請求(Get Over) + 4. 在1秒內Request Post 2個請求(Post Over) + +![image](/quiz/01-rate-limit/quiz01//image/django_test.png "This is a sample image.") + +### Locust +> [Locust](https://locust.io/) +Define user behaviour with Python code, and swarm your system with millions of simultaneous users. +可以透過Locust簡易的Loading test設定，並觀察測試數據。 + +![image](/quiz/01-rate-limit/quiz01//image/locust_index.png "This is a sample image.") +![image](/quiz/01-rate-limit/quiz01//image/locust_statistics.png "This is a sample image.") +![image](/quiz/01-rate-limit/quiz01//image/locust_failures.png "This is a sample image.") + + diff --git a/quiz/01-rate-limit/quiz01/core/__init__.py b/quiz/01-rate-limit/quiz01/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quiz/01-rate-limit/quiz01/core/admin.py b/quiz/01-rate-limit/quiz01/core/admin.py new file mode 100644 index 0000000..ea5d68b --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/quiz/01-rate-limit/quiz01/core/apps.py b/quiz/01-rate-limit/quiz01/core/apps.py new file mode 100644 index 0000000..ee53ca6 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'core' diff --git a/quiz/01-rate-limit/quiz01/core/migrations/0001_initial.py b/quiz/01-rate-limit/quiz01/core/migrations/0001_initial.py new file mode 100644 index 0000000..44d8fa5 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/migrations/0001_initial.py @@ -0,0 +1,12 @@ +# Generated by Django 3.2.18 on 2023-03-21 18:23 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ] + + operations = [ + ] diff --git a/quiz/01-rate-limit/quiz01/core/migrations/0002_customer_id_model_headerfield_model.py b/quiz/01-rate-limit/quiz01/core/migrations/0002_customer_id_model_headerfield_model.py new file mode 100644 index 0000000..27e0878 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/migrations/0002_customer_id_model_headerfield_model.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.18 on 2023-03-21 18:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('core', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Customer_ID_Model', + fields=[ + ('customer_ID', models.TextField(primary_key=True, serialize=False)), + ], + options={ + 'db_table': 'Customer_ID_Model', + }, + ), + migrations.CreateModel( + name='HeaderField_Model', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('request_type', models.TextField()), + ('Limit', models.IntegerField(default=100)), + ('Remaining', models.IntegerField(default=100)), + ('Reset', models.IntegerField(default=1)), + ('RetryAt', models.IntegerField(default=0)), + ('upData', models.DateTimeField(auto_now=True)), + ('ID', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.customer_id_model')), + ], + options={ + 'db_table': 'HeaderField_Model', + }, + ), + ] diff --git a/quiz/01-rate-limit/quiz01/core/migrations/0003_auto_20230322_0444.py b/quiz/01-rate-limit/quiz01/core/migrations/0003_auto_20230322_0444.py new file mode 100644 index 0000000..536759a --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/migrations/0003_auto_20230322_0444.py @@ -0,0 +1,51 @@ +# Generated by Django 3.2.18 on 2023-03-22 04:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_customer_id_model_headerfield_model'), + ] + + operations = [ + migrations.CreateModel( + name='GET_Model', + fields=[ + ('customer_ID', models.TextField(primary_key=True, serialize=False)), + ('Limit', models.IntegerField(default=100)), + ('Remaining', models.IntegerField(default=100)), + ('Reset', models.IntegerField(default=1)), + ('RetryAt', models.IntegerField(default=0)), + ('upData', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'GET_Model', + }, + ), + migrations.CreateModel( + name='POST_Model', + fields=[ + ('customer_ID', models.TextField(primary_key=True, serialize=False)), + ('Limit', models.IntegerField(default=1)), + ('Remaining', models.IntegerField(default=1)), + ('Reset', models.IntegerField(default=1)), + ('RetryAt', models.IntegerField(default=0)), + ('upData', models.DateTimeField(auto_now=True)), + ], + options={ + 'db_table': 'POST_Model', + }, + ), + migrations.RemoveField( + model_name='headerfield_model', + name='ID', + ), + migrations.DeleteModel( + name='Customer_ID_Model', + ), + migrations.DeleteModel( + name='HeaderField_Model', + ), + ] diff --git a/quiz/01-rate-limit/quiz01/core/migrations/0004_auto_20230413_0734.py b/quiz/01-rate-limit/quiz01/core/migrations/0004_auto_20230413_0734.py new file mode 100644 index 0000000..31107f4 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/migrations/0004_auto_20230413_0734.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.18 on 2023-04-13 07:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0003_auto_20230322_0444'), + ] + + operations = [ + migrations.AddField( + model_name='get_model', + name='path', + field=models.TextField(default='n/a'), + ), + migrations.AddField( + model_name='post_model', + name='path', + field=models.TextField(default='n/a'), + ), + ] diff --git a/quiz/01-rate-limit/quiz01/core/migrations/__init__.py b/quiz/01-rate-limit/quiz01/core/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quiz/01-rate-limit/quiz01/core/models.py b/quiz/01-rate-limit/quiz01/core/models.py new file mode 100644 index 0000000..3763f61 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/models.py @@ -0,0 +1,31 @@ +from django.db import models + +# Create your models here. + +# class Customer_ID_Model(models.Model): +# customer_ID = models.TextField(primary_key=True) +# # other customer info... +# class Meta: +# db_table = "Customer_ID_Model" + +class GET_Model(models.Model): + customer_ID = models.TextField(primary_key=True) + path = models.TextField(default = 'n/a') + Limit = models.IntegerField(default=100) + Remaining = models.IntegerField(default=100) + Reset = models.IntegerField(default=1) + RetryAt = models.IntegerField(default=0) + upData = models.DateTimeField(auto_now=True) + class Meta: + db_table = "GET_Model" + +class POST_Model(models.Model): + customer_ID = models.TextField(primary_key=True) + path = models.TextField(default = 'n/a') + Limit = models.IntegerField(default=1) + Remaining = models.IntegerField(default=1) + Reset = models.IntegerField(default=1) + RetryAt = models.IntegerField(default=0) + upData = models.DateTimeField(auto_now=True) + class Meta: + db_table = "POST_Model" diff --git a/quiz/01-rate-limit/quiz01/core/tests.py b/quiz/01-rate-limit/quiz01/core/tests.py new file mode 100644 index 0000000..d32ebb3 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/tests.py @@ -0,0 +1,204 @@ +from django.test import TestCase +from django.test import client +import time +# Create your tests here. + +class APITestCase(TestCase): + + def setUp(self): + self.c = client.Client() + + # # Unover rate limiting get test + # # will be get 200 status + def test_get_not_over(self): + + # If the execution time is greater than 1 second, + # we must retest until it is less than 1 second or has been executed 5 times + for i in range(1,5): + #reset api limit + resp = self.c.post('/reset/',{'group':'get', 'key':'ip', 'rate':'100/s', 'method':'GET'}) + t0 = int(time.time()) + #number of request + for j in range(1,101): + resp = self.c.get('/api/') #request api + + t1 = int(time.time()) + + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #we can't get it for less than 1s + else: + self.assertEqual(resp.status_code, 200) + + # # Over rate limiting get test + # # will be get 429 status + def test_get_over100(self): + + # If the execution time is greater than 1 second, + # we must retest until it is less than 1 second or has been executed 5 times + for i in range(1,5): + #reset api limit + self.c.post('/reset/',{'group':'get', 'key':'ip', 'rate':'100/s', 'method':'GET'}) + t0 = int(time.time()) + + #number of request + for j in range(1,102): + resp = self.c.get('/api/') #request api + + t1 = int(time.time()) + + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #can't done for less than 1s + else: + self.assertEqual(resp.status_code, 429) + + + # # Unover rate limiting post test + # # will be get 200 status + def test_post_not_over(self): + + for i in range(1,5): + self.c.post('/reset/',{'group':'post', 'key':'ip', 'rate':'1/s', 'method':'POST'}) + t0 = int(time.time()) + resp = self.c.post('/api/') #request api + + t1 = int(time.time()) + + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #can't done for less than 1s + else: + self.assertEqual(resp.status_code, 200) + + # # Over rate limiting post test + # # will be get 429 status + def test_post_over1(self): + for i in range(1,5): + resp = self.c.post('/reset/',{'group':'post', 'key':'ip', 'rate':'1/s', 'method':'POST'}) + t0 = int(time.time()) + resp = self.c.post('/api/') #request api + + resp = self.c.post('/api/') + + t1 = int(time.time()) + + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #can't done for less than 1s + else: + self.assertEqual(resp.status_code, 429) + + # # Unover rate limiting get test + # # will be get '10000' + def test_get_not_over_header(self): + + # If the execution time is greater than 1 second, + # we must retest until it is less than 1 second or has been executed 5 times + for i in range(1,5): + #reset api limit + resp = self.c.post('/reset/',{'group':'get', 'key':'ip', 'rate':'100/s', 'method':'GET'}) + t0 = int(time.time()) + #number of request + for j in range(1,101): + resp = self.c.get('/api/') #request api + t1 = int(time.time()) + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #we can't get it for less than 1s + else: + # get diff header file by status code + if(resp.status_code == 200): + self.assertEqual(resp['X-RateLimit-Limit']+resp['X-RateLimit-Remaining']+resp['X-RateLimit-Reset'], '10000') + elif(resp.status_code == 429): + self.assertEqual(resp.status_code, 200) + + + # # Over rate limiting get test + # # will be get '0' + def test_get_over100_header(self): + # If the execution time is greater than 1 second, + # we must retest until it is less than 1 second or has been executed 5 times + for i in range(1,5): + #reset api limit + self.c.post('/reset/',{'group':'get', 'key':'ip', 'rate':'100/s', 'method':'GET'}) + t0 = int(time.time()) + + #number of request + for j in range(1,102): + resp = self.c.get('/api/') #request api + + t1 = int(time.time()) + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #we can't get it for less than 1s + else: + # get diff header file by status code + if(resp.status_code == 200): + self.assertEqual(resp.status_code, 429) + elif(resp.status_code == 429): + self.assertEqual(resp['Retry-At'], '0') + + + # # Unover rate limiting post test + # # will be get '100' + def test_post_not_over_header(self): + for i in range(1,5): + self.c.post('/reset/',{'group':'post', 'key':'ip', 'rate':'1/s', 'method':'POST'}) + t0 = int(time.time()) + resp = self.c.post('/api/') #request api + t1 = int(time.time()) + + # if execution time < 1, don't do again + if((t1-t0) <1): + break + + if((t1-t0) >1): + self.assertEqual(0, 1) #we can't get it for less than 1s + else: + # get diff header file by status code + if(resp.status_code == 200): + self.assertEqual(resp['X-RateLimit-Limit']+resp['X-RateLimit-Remaining']+resp['X-RateLimit-Reset'], '100') + elif(resp.status_code == 429): + self.assertEqual(resp.status_code, 200) + + # # Over rate limiting post test + # # will be get '0' + def test_post_over1_header(self): + for i in range(1,5): + resp = self.c.post('/reset/',{'group':'post', 'key':'ip', 'rate':'1/s', 'method':'POST'}) + t0 = int(time.time()) + resp = self.c.post('/api/') #request api + resp = self.c.post('/api/') #request api + + t1 = int(time.time()) + # if execution time < 1, don't do again + if((t1-t0) <1): + break + if((t1-t0) >1): + self.assertEqual(0, 1) #we can't get it for less than 1s + else: + # get diff header file by status code + if(resp.status_code == 200): + self.assertEqual(resp.status_code, 429) + elif(resp.status_code == 429): + self.assertEqual(resp['Retry-At'], '0') + \ No newline at end of file diff --git a/quiz/01-rate-limit/quiz01/core/views.py b/quiz/01-rate-limit/quiz01/core/views.py new file mode 100644 index 0000000..e64a7e5 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/core/views.py @@ -0,0 +1,176 @@ +# Create your views here. + +from django.http import HttpResponse,HttpResponseForbidden +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator +from django.views.generic import View +from core.models import GET_Model, POST_Model + +import functools +from django.conf import settings +from django.core.cache import caches +from django.core.exceptions import ImproperlyConfigured +from django.utils.module_loading import import_string + +# django_ratelimit +from django_ratelimit.exceptions import Ratelimited +from django_ratelimit.decorators import ratelimit +from django_ratelimit.core import get_usage, _SIMPLE_KEYS, _ACCESSOR_KEYS, _get_window, _make_cache_key, _split_rate +from django_ratelimit import ALL, UNSAFE + + +@method_decorator(csrf_exempt, name='dispatch') +class RateLimitAPI(View): + @method_decorator(ratelimit(group = 'get',key='ip', rate='100/s', method='GET')) + def get(self,request): + block_info = get_usage(group = 'get', request=request ,key='ip', rate='100/s', method='GET', increment =False) + response = ratelimit_response(block_info) + headerfiled_get_db(request,block_info) + + return response + + @method_decorator(ratelimit(group = 'post', key='ip', rate='1/s', method='POST')) + def post(self,request): + block_info = get_usage(group = 'post', request=request ,key='ip', rate='1/s', method='POST', increment =False) + response = ratelimit_response(block_info) + headerfiled_post_db(request,block_info) + + return response + +def reset(request, group=None, fn=None, key=None, rate=None, method=ALL, increment=False): + group = request.POST['group'] + key = request.POST['key'] + rate = request.POST['rate'] + method = request.POST['method'] + if group is None and fn is None: + raise ImproperlyConfigured('get_usage must be called with either ' + '`group` or `fn` arguments') + + if not getattr(settings, 'RATELIMIT_ENABLE', True): + return None + + if group is None: + parts = [] + + if isinstance(fn, functools.partial): + fn = fn.func + + # if fn.__name__ == 'bound_func': + # fn = fn.__closure__[0].cell_contents + + if hasattr(fn, '__module__'): + parts.append(fn.__module__) + + if hasattr(fn, '__self__'): + parts.append(fn.__self__.__class__.__name__) + + # parts.append(fn.__qualname__) + group = '.'.join(parts) + + if callable(rate): + rate = rate(group, request) + elif isinstance(rate, str) and '.' in rate: + ratefn = import_string(rate) + rate = ratefn(group, request) + + if rate is None: + return HttpResponse('Ratelimit rate is None)',status=200) + limit, period = _split_rate(rate) + if period <= 0: + raise ImproperlyConfigured('Ratelimit period must be greater than 0') + + if not key: + raise ImproperlyConfigured('Ratelimit key must be specified') + if callable(key): + value = key(group, request) + elif key in _SIMPLE_KEYS: + value = _SIMPLE_KEYS[key](request) + elif ':' in key: + accessor, k = key.split(':', 1) + if accessor not in _ACCESSOR_KEYS: + raise ImproperlyConfigured('Unknown ratelimit key: %s' % key) + value = _ACCESSOR_KEYS[accessor](request, k) + elif '.' in key: + keyfn = import_string(key) + value = keyfn(group, request) + else: + raise ImproperlyConfigured( + 'Could not understand ratelimit key: %s' % key) + + window = _get_window(value, period) + + cache_name = getattr(settings, 'RATELIMIT_USE_CACHE', 'default') + cache = caches[cache_name] + cache_key = _make_cache_key(group, window, rate, value, method) + cache.get(cache_key) + if cache.delete(cache_key) == True: + return HttpResponse('Reseted',status=205) + return HttpResponse('An error occurred from reset',status=400) + + +#build ratelimit response +def ratelimit_response(block_info): + response = HttpResponse('X-RateLimit-Limit: '+str(block_info['limit'])+'\n' + + 'X-RateLimit-Remaining: '+ str(block_info['limit'] - block_info['count'] )+'\n' + + 'X-RateLimit-Reset: '+ str(block_info['time_left'])) + #header filed + response['X-RateLimit-Limit'] = block_info['limit'] + response['X-RateLimit-Remaining'] = block_info['limit'] - block_info['count'] + response['X-RateLimit-Reset'] = block_info['time_left'] + # print(block_info) + return response + +def headerfiled_post_db(request,block_info): + client_id = get_client_ip_address(request) + POST_Model.objects.update_or_create( + customer_ID = client_id, + defaults=dict( + path = request.path, + Remaining = block_info['limit'] - block_info['count'], + Reset = block_info['time_left'], + RetryAt = block_info['time_left'] + ) + ) + + +def headerfiled_get_db(request,block_info): + client_id = get_client_ip_address(request) + GET_Model.objects.update_or_create( + customer_ID = client_id, + defaults=dict( + path = request.path, + Remaining = block_info['limit'] - block_info['count'], + Reset = block_info['time_left'], + RetryAt = block_info['time_left'] + ) + ) + + + +def get_client_ip_address(request): + req_headers = request.META + x_forwarded_for_value = req_headers.get('HTTP_X_FORWARDED_FOR') + if x_forwarded_for_value: + ip_addr = x_forwarded_for_value.split(',')[-1].strip() + else: + ip_addr = req_headers.get('REMOTE_ADDR') + return ip_addr + +#if over rate limit will redirect 403 to 429 +def handler403(request, exception=None): + if isinstance(exception, Ratelimited): + print("Over Rating") + if request.method == 'POST': + block_info = get_usage(group = 'post', request=request ,key='ip', rate='1/s', method='POST', increment =False) + elif request.method == 'GET': + block_info = get_usage(group = 'get', request=request ,key='ip', rate='100/s', method='GET', increment =False) + headerfiled_get_db(request,block_info) + response = HttpResponse('Too Many Requests'+'\n' + + 'Retry-At: ' + str(block_info['time_left']), status=429) + response['Retry-At'] = block_info['time_left'] + return response + return HttpResponseForbidden('Forbidden') + + + + diff --git a/quiz/01-rate-limit/quiz01/doc/feedback_2023_0324.md b/quiz/01-rate-limit/quiz01/doc/feedback_2023_0324.md new file mode 100644 index 0000000..e69de29 diff --git a/quiz/01-rate-limit/quiz01/docker-compose.yml b/quiz/01-rate-limit/quiz01/docker-compose.yml new file mode 100644 index 0000000..4263328 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/docker-compose.yml @@ -0,0 +1,47 @@ +services: + # db: + # image: postgres + # volumes: + # - ./data/db:/var/lib/postgresql/data + # environment: + # - POSTGRES_DB=postgres + # - POSTGRES_USER=postgres + # - POSTGRES_PASSWORD=postgres + web: + build: . + image: v3.3.1 + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/code + ports: + - "8000:8000" + environment: + - POSTGRES_NAME=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + depends_on: + # - db + - migration + + migration: + image: v3.3.1 + command: bash -c docker-compose run web python3 manage.py makemigrations && python manage.py migrate + volumes: + - .:/code + # depends_on: + # - db + + # Locust + master: + image: locustio/locust + ports: + - "8089:8089" + volumes: + - ./:/mnt/locust + command: -f /mnt/locust/locustfile.py --master -H http://web:8000 + + worker: + image: locustio/locust + volumes: + - ./:/mnt/locust + command: -f /mnt/locust/locustfile.py --worker --master-host master \ No newline at end of file diff --git a/quiz/01-rate-limit/quiz01/image/django_test.png b/quiz/01-rate-limit/quiz01/image/django_test.png new file mode 100644 index 0000000..fdcf8fc Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/django_test.png differ diff --git a/quiz/01-rate-limit/quiz01/image/locust_failures.png b/quiz/01-rate-limit/quiz01/image/locust_failures.png new file mode 100644 index 0000000..acfa664 Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/locust_failures.png differ diff --git a/quiz/01-rate-limit/quiz01/image/locust_index.png b/quiz/01-rate-limit/quiz01/image/locust_index.png new file mode 100644 index 0000000..ca281be Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/locust_index.png differ diff --git a/quiz/01-rate-limit/quiz01/image/locust_statistics.png b/quiz/01-rate-limit/quiz01/image/locust_statistics.png new file mode 100644 index 0000000..b04ddc8 Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/locust_statistics.png differ diff --git a/quiz/01-rate-limit/quiz01/image/postman_429.png b/quiz/01-rate-limit/quiz01/image/postman_429.png new file mode 100644 index 0000000..73d60c5 Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/postman_429.png differ diff --git a/quiz/01-rate-limit/quiz01/image/postman_get.png b/quiz/01-rate-limit/quiz01/image/postman_get.png new file mode 100644 index 0000000..fecd672 Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/postman_get.png differ diff --git a/quiz/01-rate-limit/quiz01/image/postman_post.png b/quiz/01-rate-limit/quiz01/image/postman_post.png new file mode 100644 index 0000000..5d72f7c Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/postman_post.png differ diff --git a/quiz/01-rate-limit/quiz01/image/webpage.png b/quiz/01-rate-limit/quiz01/image/webpage.png new file mode 100644 index 0000000..c39ae31 Binary files /dev/null and b/quiz/01-rate-limit/quiz01/image/webpage.png differ diff --git a/quiz/01-rate-limit/quiz01/locustfile.py b/quiz/01-rate-limit/quiz01/locustfile.py new file mode 100644 index 0000000..f5362e7 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/locustfile.py @@ -0,0 +1,12 @@ +from locust import HttpUser, task, between + +class HelloWorldUser(HttpUser): + wait_time = between(0.5, 2.5) + + @task + def get_world(self): + self.client.get('/api/') + + @task + def post_world(self): + self.client.post('/api/') \ No newline at end of file diff --git a/quiz/01-rate-limit/quiz01/manage.py b/quiz/01-rate-limit/quiz01/manage.py new file mode 100644 index 0000000..695b682 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'quiz01.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/quiz/01-rate-limit/quiz01/quiz01/__init__.py b/quiz/01-rate-limit/quiz01/quiz01/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/quiz/01-rate-limit/quiz01/quiz01/asgi.py b/quiz/01-rate-limit/quiz01/quiz01/asgi.py new file mode 100644 index 0000000..672f50c --- /dev/null +++ b/quiz/01-rate-limit/quiz01/quiz01/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for quiz01 project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'quiz01.settings') + +application = get_asgi_application() diff --git a/quiz/01-rate-limit/quiz01/quiz01/settings.py b/quiz/01-rate-limit/quiz01/quiz01/settings.py new file mode 100644 index 0000000..c2b482c --- /dev/null +++ b/quiz/01-rate-limit/quiz01/quiz01/settings.py @@ -0,0 +1,127 @@ +""" +Django settings for quiz01 project. + +Generated by 'django-admin startproject' using Django 3.2.18. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.2/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-0%7v$ntyuann7o=z*a8zx=*-4#qc86&mfu189%zx$j6twklwga' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['testserver', + 'localhost', + 'web'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'core', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'quiz01.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'quiz01.wsgi.application' + +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ + +STATIC_URL = '/static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/quiz/01-rate-limit/quiz01/quiz01/urls.py b/quiz/01-rate-limit/quiz01/quiz01/urls.py new file mode 100644 index 0000000..12efeff --- /dev/null +++ b/quiz/01-rate-limit/quiz01/quiz01/urls.py @@ -0,0 +1,28 @@ +"""quiz01 URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path +from core.views import RateLimitAPI,reset + +from django.conf.urls import ( + handler403 +) +urlpatterns = [ + path('admin/', admin.site.urls), + path('api/',RateLimitAPI.as_view()), + path('reset/', reset) +] +handler403 = 'core.views.handler403' \ No newline at end of file diff --git a/quiz/01-rate-limit/quiz01/quiz01/wsgi.py b/quiz/01-rate-limit/quiz01/quiz01/wsgi.py new file mode 100644 index 0000000..e9beaa5 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/quiz01/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for quiz01 project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'quiz01.settings') + +application = get_wsgi_application() diff --git a/quiz/01-rate-limit/quiz01/requirements.txt b/quiz/01-rate-limit/quiz01/requirements.txt new file mode 100644 index 0000000..9729790 --- /dev/null +++ b/quiz/01-rate-limit/quiz01/requirements.txt @@ -0,0 +1,10 @@ +Django>=3.0,<4.0 +# psycopg2>=2.8 +django-ratelimit +# djangorestframework + +# test package +pyperf +locust + +markdown