quize1小作業

.gitgnore

@@ -0,0 +1 @@
+ quiz/01-rate-limit/quiz01/data/

quiz/01-rate-limit/quiz01/Dockerfile

@@ -0,0 +1,10 @@
+# syntax=docker/dockerfile:1
+FROM python:3
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+WORKDIR /code
+COPY requirements.txt /code/
+COPY Pytest_test.py /code/
+COPY locustfile.py /code/
+RUN pip install -r requirements.txt
+COPY . /code/

quiz/01-rate-limit/quiz01/Pytest_test.py

@@ -0,0 +1,7 @@
+import pyperf
+from core.views import RateLimitAPI
+setup = "from core.views import RateLimitAPI"
+runner = pyperf.Runner()
+runner.timeit(name="Get test",
+              stmt="RateLimitAPI.get",
+              setup=setup)

quiz/01-rate-limit/quiz01/README.md

@@ -0,0 +1,63 @@
+# 01-rate-limit
+>Rate limiting is used to protect resources from being over-using or abused by users/bots/applications. It is commonly implemented by social media platforms such as Facebook or Instagram.
+## The project components
+用Django做應用框架搭配 [django-ratelimit](https://django-ratelimit.readthedocs.io/en/stable/)做速率限制，在資料庫上用Sqlite做每個使用者的header file的紀錄。
+
+
+## Usage  
+1. 建立Image，tag加入"v3.3.1"，或者自行至`docker-compose.yml`修改web的image的賦予名稱
+
+```
+docker build -t="v3.3.1" .
+```
+2. 載入[locustio/locust](https://hub.docker.com/r/locustio/locust)，Load testing 的工具
+```
+docker pull locustio/locust
+```
+
+3. 啟動container
+```
+docker-compose up --scale worker=4
+```
+4. 開啟網頁，導向 http://localhost:8000/api/  
+
+![image](/quiz/01-rate-limit/quiz01/image/webpage.png)
+
+
+## Test
+使用Postman和撰寫Django Test做基本ratelimit測試，也加入[Locust](https://locust.io/)做Load testing 和 Profiling 評估。  
+### Postman
+Postman 是常用的api測試工具，我們可以透過Postman簡單的進行Request，也可以透過該工具清楚的看到response的header field。  
+
+**Get**  
+![image](/quiz/01-rate-limit/quiz01//image/postman_get.png "This is a sample image.")   
+
+**Post**  
+![image](/quiz/01-rate-limit/quiz01//image/postman_post.png "This is a sample image.")  
+
+**429 Too Many Requests**  
+![image](/quiz/01-rate-limit/quiz01//image/postman_429.png "This is a sample image.")  
+
+### Django Test
+透過Django Test 可以自訂義test的方法，並觀察1秒內的Request count，超過後顯示"Over Rating"。  
+```
+docker-compose run web python3 manage.py test
+```
+* 自訂義Django Test 4種測試:  
+    1. 在1秒內Request Get 100個請求(Get Not Over)  
+    2. 在1秒內Request Post 1個請求(Post Not Over)  
+    3. 在1秒內Request Get 101個請求(Get Over)  
+    4. 在1秒內Request Post 2個請求(Post Over)  
+
+![image](/quiz/01-rate-limit/quiz01//image/django_test.png "This is a sample image.")
+
+### Locust
+> [Locust](https://locust.io/)  
+Define user behaviour with Python code, and swarm your system with millions of simultaneous users.  
+可以透過Locust簡易的Loading test設定，並觀察測試數據。  
+
+![image](/quiz/01-rate-limit/quiz01//image/locust_index.png "This is a sample image.")  
+![image](/quiz/01-rate-limit/quiz01//image/locust_statistics.png "This is a sample image.")  
+![image](/quiz/01-rate-limit/quiz01//image/locust_failures.png "This is a sample image.")  
+
+

quiz/01-rate-limit/quiz01/core/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

quiz/01-rate-limit/quiz01/core/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'core'

quiz/01-rate-limit/quiz01/core/migrations/0001_initial.py

@@ -0,0 +1,12 @@
+# Generated by Django 3.2.18 on 2023-03-21 18:23
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+    ]

quiz/01-rate-limit/quiz01/core/migrations/0002_customer_id_model_headerfield_model.py

@@ -0,0 +1,41 @@
+# Generated by Django 3.2.18 on 2023-03-21 18:28
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Customer_ID_Model',
+            fields=[
+                ('customer_ID', models.TextField(primary_key=True, serialize=False)),
+            ],
+            options={
+                'db_table': 'Customer_ID_Model',
+            },
+        ),
+        migrations.CreateModel(
+            name='HeaderField_Model',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('request_type', models.TextField()),
+                ('Limit', models.IntegerField(default=100)),
+                ('Remaining', models.IntegerField(default=100)),
+                ('Reset', models.IntegerField(default=1)),
+                ('RetryAt', models.IntegerField(default=0)),
+                ('upData', models.DateTimeField(auto_now=True)),
+                ('ID', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.customer_id_model')),
+            ],
+            options={
+                'db_table': 'HeaderField_Model',
+            },
+        ),
+    ]

quiz/01-rate-limit/quiz01/core/migrations/0003_auto_20230322_0444.py

@@ -0,0 +1,51 @@
+# Generated by Django 3.2.18 on 2023-03-22 04:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_customer_id_model_headerfield_model'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GET_Model',
+            fields=[
+                ('customer_ID', models.TextField(primary_key=True, serialize=False)),
+                ('Limit', models.IntegerField(default=100)),
+                ('Remaining', models.IntegerField(default=100)),
+                ('Reset', models.IntegerField(default=1)),
+                ('RetryAt', models.IntegerField(default=0)),
+                ('upData', models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                'db_table': 'GET_Model',
+            },
+        ),
+        migrations.CreateModel(
+            name='POST_Model',
+            fields=[
+                ('customer_ID', models.TextField(primary_key=True, serialize=False)),
+                ('Limit', models.IntegerField(default=1)),
+                ('Remaining', models.IntegerField(default=1)),
+                ('Reset', models.IntegerField(default=1)),
+                ('RetryAt', models.IntegerField(default=0)),
+                ('upData', models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                'db_table': 'POST_Model',
+            },
+        ),
+        migrations.RemoveField(
+            model_name='headerfield_model',
+            name='ID',
+        ),
+        migrations.DeleteModel(
+            name='Customer_ID_Model',
+        ),
+        migrations.DeleteModel(
+            name='HeaderField_Model',
+        ),
+    ]

quiz/01-rate-limit/quiz01/core/models.py

@@ -0,0 +1,29 @@
+from django.db import models
+
+# Create your models here.
+
+# class Customer_ID_Model(models.Model):
+#     customer_ID = models.TextField(primary_key=True)
+#     # other customer info...
+#     class Meta:
+#         db_table = "Customer_ID_Model"
+
+class GET_Model(models.Model):
+    customer_ID = models.TextField(primary_key=True)
+    Limit = models.IntegerField(default=100)
+    Remaining = models.IntegerField(default=100)
+    Reset = models.IntegerField(default=1)
+    RetryAt = models.IntegerField(default=0)
+    upData = models.DateTimeField(auto_now=True)
+    class Meta:
+        db_table = "GET_Model"
+
+class POST_Model(models.Model):
+    customer_ID = models.TextField(primary_key=True)
+    Limit = models.IntegerField(default=1)
+    Remaining = models.IntegerField(default=1)
+    Reset = models.IntegerField(default=1)
+    RetryAt = models.IntegerField(default=0)
+    upData = models.DateTimeField(auto_now=True)
+    class Meta:
+        db_table = "POST_Model"

quiz/01-rate-limit/quiz01/core/tests.py

@@ -0,0 +1,47 @@
+from django.test import TestCase
+from django.test import client
+import time
+# Create your tests here.
+
+class APITestCase(TestCase):
+
+    def setUp(self):
+        self.c = client.Client()
+
+    # Unover rate limiting get test 
+    # will be get 200 status
+    def test_get_not_over(self):
+        time.sleep(2)
+        print('-----------Get not over-----------')
+        for i in range(1,101):
+            resp = self.c.get('/api/')
+        self.assertEqual(resp.status_code, 200)
+
+
+    # Unover rate limiting post test 
+    # will be get 200 status
+    def test_post_not_over(self):
+        time.sleep(1)
+        print('-----------Post not over-----------')
+        resp = self.c.post('/api/')
+        self.assertEqual(resp.status_code, 200)
+
+    # Over rate limiting get test 
+    # will be get 429 status
+
+    def test_get_over100(self):
+        time.sleep(2)
+        print('-----------Get over-----------')
+        for i in range(1,102):
+            resp = self.c.get('/api/')
+        self.assertEqual(resp.status_code, 429)
+
+    # # Over rate limiting post test 
+    # # # will be get 429 status
+    def test_post_over1(self):
+        time.sleep(1)
+        print('-----------Post over-----------')
+        resp = self.c.post('/api/')
+        resp = self.c.post('/api/')
+        self.assertEqual(resp.status_code, 429)    
+

quiz/01-rate-limit/quiz01/core/views.py

@@ -0,0 +1,108 @@
+# Create your views here.
+
+# from core.models import CoreModel
+from django_ratelimit.decorators import ratelimit
+from django.http import HttpResponse,HttpResponseForbidden
+from django.views.decorators.csrf import csrf_exempt
+from django.utils.decorators import method_decorator
+from django.views.generic import View
+from django_ratelimit.exceptions import Ratelimited
+from django_ratelimit.core import get_usage, is_ratelimited
+from core.models import GET_Model, POST_Model
+
+@method_decorator(csrf_exempt, name='dispatch')
+class RateLimitAPI(View):
+
+    @classmethod
+    @method_decorator(ratelimit(key='ip', rate='100/s', method='GET'))
+    def get(cls, request):
+        block_info = ratelimit_tracking(cls,request,'100/s')
+        headerfiled_get_db(request,block_info)
+        response = HttpResponse('X-RateLimit-Limit: '+str(block_info['limit'])+'\n'
+                                + 'X-RateLimit-Remaining: '+ str(block_info['limit'] - block_info['count'] )+'\n'
+                                + 'X-RateLimit-Reset: '+ str(block_info['time_left']))
+        #header filed
+        response['X-RateLimit-Limit'] = block_info['limit']
+        response['X-RateLimit-Remaining'] = block_info['limit'] - block_info['count']
+        response['X-RateLimit-Reset'] = block_info['time_left']
+        return response
+
+    @classmethod
+    @method_decorator(ratelimit(key='ip', rate='1/s', method='Post'))
+    def post(cls, request):
+        block_info = ratelimit_tracking(cls,request,'1/s')
+        headerfiled_post_db(request,block_info)
+        response = HttpResponse('X-RateLimit-Limit: '+str(block_info['limit'])+'\n'
+                                + 'X-RateLimit-Remaining: '+ str(block_info['limit'] - block_info['count'] )+'\n'
+                                + 'X-RateLimit-Reset: '+ str(block_info['time_left']))
+        #header filed
+        response['X-RateLimit-Limit'] = block_info['limit']
+        response['X-RateLimit-Remaining'] = block_info['limit'] - block_info['count']
+        response['X-RateLimit-Reset'] = block_info['time_left']
+        return response
+
+#get ratelimit info
+def ratelimit_tracking(fun,request,fun_rate):
+    block_info = get_usage(request, key="ip",fn=fun, rate=fun_rate,increment =True)
+    print(block_info)
+    return block_info
+
+def headerfiled_post_db(request,block_info):
+    client_id = get_client_ip_address(request)
+    if POST_Model.objects.filter(customer_ID=client_id).exists():
+        db_info = POST_Model.objects.filter(customer_ID = client_id)
+        db_info.update(Limit = block_info['limit'],
+                       Remaining = block_info['limit'] - block_info['count'],
+                       Reset =block_info['time_left'],
+                       RetryAt = block_info['time_left'])
+
+    else:
+        POST_Model.objects.create(customer_ID = client_id,
+                                  Limit = block_info['limit'],
+                                  Remaining = block_info['limit'] - block_info['count'],
+                                  Reset =block_info['time_left'],
+                                  RetryAt = block_info['time_left'])
+
+def headerfiled_get_db(request,block_info):
+    client_id = get_client_ip_address(request)
+    if GET_Model.objects.filter(customer_ID=client_id).exists():
+        db_info = GET_Model.objects.filter(customer_ID = client_id)
+        db_info.update(Limit = block_info['limit'],
+                       Remaining = block_info['limit'] - block_info['count'],
+                       Reset =block_info['time_left'],
+                       RetryAt = block_info['time_left'])
+
+    else:
+        GET_Model.objects.create(customer_ID = client_id,
+                                  Limit = block_info['limit'],
+                                  Remaining = block_info['limit'] - block_info['count'],
+                                  Reset =block_info['time_left'],
+                                  RetryAt = block_info['time_left'])
+
+def get_client_ip_address(request):
+    req_headers = request.META
+    x_forwarded_for_value = req_headers.get('HTTP_X_FORWARDED_FOR')
+    if x_forwarded_for_value:
+        ip_addr = x_forwarded_for_value.split(',')[-1].strip()
+    else:
+        ip_addr = req_headers.get('REMOTE_ADDR')
+    return ip_addr
+
+#if over rate limit will redirect 403 to 429
+def handler403(request, exception=None):
+    if isinstance(exception, Ratelimited):
+        print("Over Rating")
+        if request.method == 'POST':
+            block_info = ratelimit_tracking(RateLimitAPI.post,request,'1/s')
+        elif request.method == 'GET':
+            block_info = ratelimit_tracking(RateLimitAPI.get,request,'100/s')
+        headerfiled_get_db(request,block_info)
+        response = HttpResponse('Too Many Requests'+'\n'
+                                + 'Retry-At: ' + str(block_info['time_left']), status=429)
+        response['Retry-At'] = block_info['time_left']
+        return response
+    return HttpResponseForbidden('Forbidden')    
+
+
+
+

quiz/01-rate-limit/quiz01/data/db/PG_VERSION

@@ -0,0 +1 @@
+15