Skip to content

Commit bb73e24

Browse files
NeibceNeibce
authored
#540 쿠버네티스 관측성 확보 후 후속 오류 수정 2 (#710)
* Loki 대시보드 datasource 고정 * 제출 throttling Redis 값 처리 수정
1 parent e967062 commit bb73e24

4 files changed

Lines changed: 48 additions & 17 deletions

File tree

‎backend/utils/tests.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import json
22
import logging
33
import os
4+
import pickle
45
from unittest.mock import patch, mock_open, MagicMock
56
from django.test import SimpleTestCase, TestCase, override_settings
67
from django.core.cache import cache
@@ -10,6 +11,26 @@
1011
from utils.observability_metrics import CodePlaceCollector
1112
from utils import observability_tracing
1213
from utils.testcase_cache import TestCaseCacheManager
14+
from utils.throttling import TokenBucket
15+
16+
17+
class TokenBucketTest(SimpleTestCase):
18+
19+
def test_reads_raw_byte_values(self):
20+
redis_conn = MagicMock()
21+
redis_conn.hget.side_effect = [b"7.0"]
22+
23+
bucket = TokenBucket("1", capacity=10, fill_rate=1, default_capacity=10, redis_conn=redis_conn)
24+
25+
self.assertEqual(bucket._last_capacity, 7.0)
26+
27+
def test_reads_pickled_byte_values_from_django_redis_hash(self):
28+
redis_conn = MagicMock()
29+
redis_conn.hget.side_effect = [pickle.dumps(7.0)]
30+
31+
bucket = TokenBucket("1", capacity=10, fill_rate=1, default_capacity=10, redis_conn=redis_conn)
32+
33+
self.assertEqual(bucket._last_capacity, 7.0)
1334

1435

1536
class CodePlaceCollectorTest(SimpleTestCase):

‎backend/utils/throttling.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import time
2+
import pickle
23

34

45
class TokenBucket:
@@ -22,6 +23,15 @@ def __init__(self, key, capacity, fill_rate, default_capacity, redis_conn):
2223
self._last_capacity_key = "last_capacity"
2324
self._last_timestamp_key = "last_timestamp"
2425

26+
@staticmethod
27+
def _as_float(value):
28+
if isinstance(value, bytes):
29+
try:
30+
value = value.decode()
31+
except UnicodeDecodeError:
32+
value = pickle.loads(value)
33+
return float(value)
34+
2535
def _init_key(self):
2636
self._last_capacity = self._default_capacity
2737
now = time.time()
@@ -34,15 +44,15 @@ def _last_capacity(self):
3444
if last_capacity is None:
3545
return self._init_key()[0]
3646
else:
37-
return float(last_capacity)
47+
return self._as_float(last_capacity)
3848

3949
@_last_capacity.setter
4050
def _last_capacity(self, value):
4151
self._redis_conn.hset(self._key, self._last_capacity_key, value)
4252

4353
@property
4454
def _last_timestamp(self):
45-
return float(self._redis_conn.hget(self._key, self._last_timestamp_key))
55+
return self._as_float(self._redis_conn.hget(self._key, self._last_timestamp_key))
4656

4757
@_last_timestamp.setter
4858
def _last_timestamp(self, value):

‎kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ data:
191191
"type": "logs",
192192
"title": "Kubernetes Warning Events",
193193
"gridPos": {"x": 0, "y": 12, "w": 24, "h": 9},
194-
"datasource": "Loki",
194+
"datasource": {"type": "loki", "uid": "Loki"},
195195
"options": {
196196
"showLabels": false,
197197
"showTime": true,
@@ -206,7 +206,7 @@ data:
206206
"type": "logs",
207207
"title": "Scheduling, Pull, Restart, and Probe Events",
208208
"gridPos": {"x": 0, "y": 21, "w": 24, "h": 9},
209-
"datasource": "Loki",
209+
"datasource": {"type": "loki", "uid": "Loki"},
210210
"options": {
211211
"showLabels": false,
212212
"showTime": true,

‎kubernetes/monitoring/grafana-dashboard-logs.yaml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ data:
106106
"type": "stat",
107107
"title": "Log Lines",
108108
"gridPos": {"x": 12, "y": 0, "w": 4, "h": 4},
109-
"datasource": "Loki",
109+
"datasource": {"type": "loki", "uid": "Loki"},
110110
"fieldConfig": {"defaults": {"unit": "short"}},
111111
"options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}},
112112
"targets": [
@@ -117,7 +117,7 @@ data:
117117
"type": "stat",
118118
"title": "Backend Errors",
119119
"gridPos": {"x": 16, "y": 0, "w": 4, "h": 4},
120-
"datasource": "Loki",
120+
"datasource": {"type": "loki", "uid": "Loki"},
121121
"fieldConfig": {
122122
"defaults": {
123123
"unit": "short",
@@ -140,7 +140,7 @@ data:
140140
"type": "stat",
141141
"title": "Frontend 5xx",
142142
"gridPos": {"x": 20, "y": 0, "w": 4, "h": 4},
143-
"datasource": "Loki",
143+
"datasource": {"type": "loki", "uid": "Loki"},
144144
"fieldConfig": {
145145
"defaults": {
146146
"unit": "short",
@@ -163,7 +163,7 @@ data:
163163
"type": "timeseries",
164164
"title": "Log Lines by Namespace",
165165
"gridPos": {"x": 0, "y": 4, "w": 8, "h": 7},
166-
"datasource": "Loki",
166+
"datasource": {"type": "loki", "uid": "Loki"},
167167
"fieldConfig": {"defaults": {"unit": "logs/sec"}},
168168
"targets": [
169169
{"expr": "sum by (namespace) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{namespace}}"}
@@ -173,7 +173,7 @@ data:
173173
"type": "timeseries",
174174
"title": "Application Log Lines by App",
175175
"gridPos": {"x": 8, "y": 4, "w": 8, "h": 7},
176-
"datasource": "Loki",
176+
"datasource": {"type": "loki", "uid": "Loki"},
177177
"fieldConfig": {"defaults": {"unit": "logs/sec"}},
178178
"targets": [
179179
{"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"}[5m]))", "legendFormat": "{{namespace}} {{app}}"},
@@ -184,7 +184,7 @@ data:
184184
"type": "timeseries",
185185
"title": "Structured Error Logs by Component",
186186
"gridPos": {"x": 16, "y": 4, "w": 8, "h": 7},
187-
"datasource": "Loki",
187+
"datasource": {"type": "loki", "uid": "Loki"},
188188
"fieldConfig": {"defaults": {"unit": "logs/sec"}},
189189
"targets": [
190190
{"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{app}}"},
@@ -195,7 +195,7 @@ data:
195195
"type": "stat",
196196
"title": "Ingress 5xx",
197197
"gridPos": {"x": 0, "y": 11, "w": 4, "h": 4},
198-
"datasource": "Loki",
198+
"datasource": {"type": "loki", "uid": "Loki"},
199199
"fieldConfig": {
200200
"defaults": {
201201
"unit": "short",
@@ -218,7 +218,7 @@ data:
218218
"type": "timeseries",
219219
"title": "Log Lines by Node",
220220
"gridPos": {"x": 0, "y": 15, "w": 4, "h": 6},
221-
"datasource": "Loki",
221+
"datasource": {"type": "loki", "uid": "Loki"},
222222
"fieldConfig": {"defaults": {"unit": "logs/sec"}},
223223
"targets": [
224224
{"expr": "sum by (node) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{node}}"}
@@ -228,7 +228,7 @@ data:
228228
"type": "logs",
229229
"title": "Recent Backend Errors",
230230
"gridPos": {"x": 4, "y": 11, "w": 10, "h": 10},
231-
"datasource": "Loki",
231+
"datasource": {"type": "loki", "uid": "Loki"},
232232
"options": {
233233
"showLabels": false,
234234
"showTime": true,
@@ -243,7 +243,7 @@ data:
243243
"type": "logs",
244244
"title": "Auth, Judge, Celery, and vLLM Logs",
245245
"gridPos": {"x": 14, "y": 11, "w": 10, "h": 10},
246-
"datasource": "Loki",
246+
"datasource": {"type": "loki", "uid": "Loki"},
247247
"options": {
248248
"showLabels": false,
249249
"showTime": true,
@@ -258,7 +258,7 @@ data:
258258
"type": "logs",
259259
"title": "Ingress and Frontend 4xx/5xx Logs",
260260
"gridPos": {"x": 0, "y": 21, "w": 24, "h": 8},
261-
"datasource": "Loki",
261+
"datasource": {"type": "loki", "uid": "Loki"},
262262
"options": {
263263
"showLabels": false,
264264
"showTime": true,
@@ -274,7 +274,7 @@ data:
274274
"type": "logs",
275275
"title": "Request ID Lookup",
276276
"gridPos": {"x": 0, "y": 29, "w": 24, "h": 9},
277-
"datasource": "Loki",
277+
"datasource": {"type": "loki", "uid": "Loki"},
278278
"options": {
279279
"showLabels": false,
280280
"showTime": true,
@@ -289,7 +289,7 @@ data:
289289
"type": "logs",
290290
"title": "Frontend Runtime Error Reports",
291291
"gridPos": {"x": 0, "y": 38, "w": 24, "h": 8},
292-
"datasource": "Loki",
292+
"datasource": {"type": "loki", "uid": "Loki"},
293293
"options": {
294294
"showLabels": false,
295295
"showTime": true,

0 commit comments

Comments
 (0)