diff --git a/.gitignore b/.gitignore index 117844b..55bf421 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,12 @@ uploads/ .env .flaskenv +# Test scripts +test_upload.py +test_b2_connection.py +diagnose_ssl.py +test_b2_insecure.py + # IDE .vscode/ .idea/ diff --git a/Makefile b/Makefile index 54fc4bf..066ace6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install run down restart test clean format lint check migrate createsuperuser +.PHONY: help install run down restart test clean format lint check migrate createsuperuser shell PYTHON := backend/venv/bin/python PYTEST := backend/venv/bin/pytest @@ -16,6 +16,7 @@ help: @echo " make check Run Django system checks" @echo " make migrate Run database migrations" @echo " make createsuperuser Create Django superuser" + @echo " make shell Open Django shell" install: cd backend && venv/bin/pip install -r ../requirements.txt @@ -54,3 +55,6 @@ migrate: createsuperuser: cd backend && ../$(PYTHON) manage.py createsuperuser + +shell: + cd backend && ../$(PYTHON) manage.py shell diff --git a/backend/config/decorators.py b/backend/config/decorators.py new file mode 100644 index 0000000..5252f29 --- /dev/null +++ b/backend/config/decorators.py @@ -0,0 +1,34 @@ +""" +Custom decorators for database error handling +""" +from functools import wraps +from django.db.utils import OperationalError, DatabaseError +from django.http import JsonResponse +from django.shortcuts import render + + +def handle_database_errors(view_func): + """ + Decorator to catch database errors in views and return graceful error responses + Use this on views that might fail if database is down + """ + @wraps(view_func) + def wrapper(request, *args, **kwargs): + try: + return view_func(request, *args, **kwargs) + except (OperationalError, DatabaseError) as e: + # Log the error + print(f"Database error in {view_func.__name__}: {e}") + + # Return appropriate error response based on request type + if request.headers.get('X-Requested-With') == 'XMLHttpRequest' or request.content_type == 'application/json': + # For AJAX/API requests, return JSON error + return JsonResponse({ + 'error': 'Database temporarily unavailable', + 'message': 'Please try again in a moment' + }, status=503) + else: + # For regular requests, render error page + return render(request, 'errors/database_error.html', status=503) + + return wrapper diff --git a/backend/config/middleware.py b/backend/config/middleware.py new file mode 100644 index 0000000..39d852f --- /dev/null +++ b/backend/config/middleware.py @@ -0,0 +1,106 @@ +""" +Custom middleware for database connection handling +""" +import time +from django.conf import settings +from django.db import connection +from django.db.utils import OperationalError +from django.http import HttpResponse +from django.shortcuts import render + + +class DatabaseHealthCheckMiddleware: + """ + Middleware to handle database connection issues gracefully with retry logic + """ + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + # Try to ensure database connection with retries + if not self._ensure_database_connection(): + # If all retries failed, show a friendly error page + return self._render_database_error_page(request) + + response = self.get_response(request) + return response + + def _ensure_database_connection(self): + """ + Attempt to connect to database with exponential backoff retries + Returns True if successful, False otherwise + """ + for attempt in range(settings.DATABASE_RETRY_ATTEMPTS): + try: + # Try a simple database query to check connection + connection.ensure_connection() + return True + except OperationalError as e: + # If this is the last attempt, give up + if attempt == settings.DATABASE_RETRY_ATTEMPTS - 1: + print(f"Database connection failed after {settings.DATABASE_RETRY_ATTEMPTS} attempts") + return False + + # Wait before retrying (exponential backoff) + delay = settings.DATABASE_RETRY_DELAYS[attempt] + print(f"Database connection attempt {attempt + 1} failed, retrying in {delay}s...") + time.sleep(delay) + + return False + + def _render_database_error_page(self, request): + """ + Render a friendly error page when database is unavailable + """ + html = """ + + + + Service Temporarily Unavailable + + + +
+

We'll be right back!

+

Our database is taking a quick nap. This usually resolves itself in a few moments.

+

Please try refreshing the page in a moment.

+ +
+ + + """ + return HttpResponse(html, status=503) diff --git a/backend/config/settings.py b/backend/config/settings.py index df25bfd..079ea35 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -59,6 +59,7 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", # Serve static files + "config.middleware.DatabaseHealthCheckMiddleware", # Database health check with retry "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -96,9 +97,14 @@ "default": dj_database_url.config( default=os.environ.get("DATABASE_URL"), conn_max_age=600, + conn_health_checks=True, # Enable connection health checks ) } +# Database connection retry settings (exponential backoff: 1s, 2s, 4s) +DATABASE_RETRY_ATTEMPTS = 3 +DATABASE_RETRY_DELAYS = [1, 2, 4] # Exponential backoff in seconds + # Password validation diff --git a/backend/listings/utils.py b/backend/listings/utils.py index 5da358b..28bbd3e 100644 --- a/backend/listings/utils.py +++ b/backend/listings/utils.py @@ -138,7 +138,7 @@ def save_picture_to_b2(form_picture): def save_picture(form_picture): - """Save picture to B2 if configured, otherwise save locally""" + """Save picture to B2 storage only - no local fallback""" # Validate file first try: validate_image_file(form_picture) @@ -146,41 +146,14 @@ def save_picture(form_picture): print(f"File validation error: {e}") return None - # Try B2 first if configured and available - if B2_AVAILABLE and settings.B2_KEY_ID: - b2_filename = save_picture_to_b2(form_picture) - if b2_filename: - return b2_filename - - # Fallback to local storage - random_hex = secrets.token_hex(8) - picture_fn = f"{random_hex}.jpg" # Always save as .jpg for smaller files - picture_path = os.path.join(settings.MEDIA_ROOT, picture_fn) - - # Create upload directory if it doesn't exist - os.makedirs(os.path.dirname(picture_path), exist_ok=True) - - # Resize and optimize image - img = Image.open(form_picture) - - # Apply EXIF orientation to prevent rotation issues - try: - from PIL import ImageOps - img = ImageOps.exif_transpose(img) - except Exception: - pass # If EXIF orientation fails, continue without it - - # Convert to RGB if necessary (for JPEG) - if img.mode in ("RGBA", "LA", "P"): - img = img.convert("RGB") - - # Resize to max 800x600 while maintaining aspect ratio - img.thumbnail((800, 600), Image.Resampling.LANCZOS) - - # Save as optimized JPEG - img.save(picture_path, format="JPEG", quality=85, optimize=True) + # B2 is required - fail if not configured + if not B2_AVAILABLE or not settings.B2_KEY_ID: + print("ERROR: B2 storage not configured") + return None - return picture_fn + # Upload to B2 - fail if it doesn't work + b2_filename = save_picture_to_b2(form_picture) + return b2_filename # Will be None if upload failed def delete_photo_from_b2(filename): diff --git a/backend/listings/views.py b/backend/listings/views.py index 79edfe0..14b40ab 100644 --- a/backend/listings/views.py +++ b/backend/listings/views.py @@ -157,22 +157,20 @@ def user_login(request): if request.method == "POST": form = LoginForm(request, data=request.POST) if form.is_valid(): - email = form.cleaned_data.get("username") - password = form.cleaned_data.get("password") - user = authenticate(request, username=email, password=password) - if user is not None: - login(request, user) - messages.success(request, f"Welcome back, {user.first_name}!") - - # Validate next parameter to prevent open redirects - next_url = request.GET.get("next", "index") - if url_has_allowed_host_and_scheme( - url=next_url, - allowed_hosts={request.get_host()}, - require_https=request.is_secure() - ): - return redirect(next_url) - return redirect("index") + # AuthenticationForm validates credentials automatically + user = form.get_user() + login(request, user) + messages.success(request, f"Welcome back, {user.first_name}!") + + # Validate next parameter to prevent open redirects + next_url = request.GET.get("next", "index") + if url_has_allowed_host_and_scheme( + url=next_url, + allowed_hosts={request.get_host()}, + require_https=request.is_secure() + ): + return redirect(next_url) + return redirect("index") else: form = LoginForm() diff --git a/backend/templates/errors/database_error.html b/backend/templates/errors/database_error.html new file mode 100644 index 0000000..d603f26 --- /dev/null +++ b/backend/templates/errors/database_error.html @@ -0,0 +1,113 @@ + + + + + + Service Temporarily Unavailable - VedgyProject + + + +
+
💤
+

We'll be right back!

+

Our database is taking a quick nap. This usually resolves itself in a few moments.

+

If you're seeing this, it means our service is temporarily experiencing connection issues.

+ +
+ ← Return to Home +
+ + + + diff --git a/backend/templates/login.html b/backend/templates/login.html index f367cac..03eb464 100644 --- a/backend/templates/login.html +++ b/backend/templates/login.html @@ -4,10 +4,18 @@

Sign In

- +
{% csrf_token %} + {% if form.non_field_errors %} +
+ {% for error in form.non_field_errors %} +

{{ error }}

+ {% endfor %} +
+ {% endif %} +
{{ form.username }} diff --git a/backend/tests/test_database_resilience.py b/backend/tests/test_database_resilience.py new file mode 100644 index 0000000..2f2579b --- /dev/null +++ b/backend/tests/test_database_resilience.py @@ -0,0 +1,266 @@ +"""Tests for database resilience features""" + +import time +from unittest.mock import Mock, patch + +import pytest +from django.conf import settings +from django.db import connection +from django.db.utils import OperationalError +from django.test import Client, RequestFactory + +from config.middleware import DatabaseHealthCheckMiddleware + + +@pytest.mark.django_db +class TestDatabaseHealthCheckMiddleware: + """Test database health check middleware""" + + def test_middleware_allows_request_when_database_is_healthy(self): + """Test that requests pass through when database is working""" + factory = RequestFactory() + request = factory.get("/") + + get_response = Mock(return_value="Response") + middleware = DatabaseHealthCheckMiddleware(get_response) + + response = middleware(request) + + assert response == "Response" + get_response.assert_called_once_with(request) + + @patch('config.middleware.connection.ensure_connection') + def test_middleware_retries_on_database_failure(self, mock_ensure_connection): + """Test that middleware retries connection on failure""" + # Simulate database failure then success + mock_ensure_connection.side_effect = [ + OperationalError("Connection failed"), + None, # Success on second try + ] + + factory = RequestFactory() + request = factory.get("/") + get_response = Mock(return_value="Response") + middleware = DatabaseHealthCheckMiddleware(get_response) + + with patch('time.sleep'): # Skip actual sleep delays + response = middleware(request) + + # Should succeed after retry + assert response == "Response" + assert mock_ensure_connection.call_count == 2 + + @patch('config.middleware.connection.ensure_connection') + def test_middleware_returns_error_page_after_max_retries(self, mock_ensure_connection): + """Test that middleware returns 503 error after all retries fail""" + # Simulate continuous database failure + mock_ensure_connection.side_effect = OperationalError("Connection failed") + + factory = RequestFactory() + request = factory.get("/") + get_response = Mock(return_value="Response") + middleware = DatabaseHealthCheckMiddleware(get_response) + + with patch('time.sleep'): # Skip actual sleep delays + response = middleware(request) + + # Should return error page + assert response.status_code == 503 + assert b"We'll be right back!" in response.content + # Should have tried the configured number of attempts + assert mock_ensure_connection.call_count == settings.DATABASE_RETRY_ATTEMPTS + + @patch('config.middleware.connection.ensure_connection') + def test_middleware_uses_exponential_backoff(self, mock_ensure_connection): + """Test that middleware uses correct retry delays""" + mock_ensure_connection.side_effect = OperationalError("Connection failed") + + factory = RequestFactory() + request = factory.get("/") + get_response = Mock(return_value="Response") + middleware = DatabaseHealthCheckMiddleware(get_response) + + with patch('time.sleep') as mock_sleep: + response = middleware(request) + + # Should use the configured delays (1s, 2s, 4s) + assert mock_sleep.call_count == settings.DATABASE_RETRY_ATTEMPTS - 1 + # Check that delays match settings (excluding the last attempt which doesn't sleep) + expected_delays = settings.DATABASE_RETRY_DELAYS[:settings.DATABASE_RETRY_ATTEMPTS - 1] + actual_delays = [call.args[0] for call in mock_sleep.call_args_list] + assert actual_delays == expected_delays + + +@pytest.mark.django_db +class TestDatabaseConnectionSettings: + """Test database connection configuration""" + + def test_database_has_connection_health_checks_enabled(self): + """Test that conn_health_checks is enabled""" + db_config = settings.DATABASES['default'] + assert db_config.get('CONN_HEALTH_CHECKS') is True + + def test_database_has_connection_max_age(self): + """Test that conn_max_age is configured""" + db_config = settings.DATABASES['default'] + assert db_config.get('CONN_MAX_AGE') == 600 + + def test_retry_settings_are_configured(self): + """Test that retry settings exist and are valid""" + assert hasattr(settings, 'DATABASE_RETRY_ATTEMPTS') + assert hasattr(settings, 'DATABASE_RETRY_DELAYS') + assert settings.DATABASE_RETRY_ATTEMPTS == 3 + assert settings.DATABASE_RETRY_DELAYS == [1, 2, 4] + assert len(settings.DATABASE_RETRY_DELAYS) >= settings.DATABASE_RETRY_ATTEMPTS - 1 + + +@pytest.mark.django_db +class TestDatabaseConnectionRecovery: + """Test that database operations recover from temporary failures""" + + def test_database_connection_actually_works(self): + """Baseline test: verify database is working normally""" + from users.models import User + + # Should be able to query database + users = User.objects.all() + assert users is not None + + @patch('django.db.backends.base.base.BaseDatabaseWrapper.ensure_connection') + def test_view_handles_database_error_gracefully(self, mock_ensure_connection): + """Test that views handle database errors without crashing""" + # Simulate a temporary database issue + mock_ensure_connection.side_effect = [ + OperationalError("Connection refused"), + None, # Recover on retry + ] + + client = Client() + + with patch('time.sleep'): # Skip delays + response = client.get("/") + + # Should get a response (either success or graceful error) + assert response.status_code in [200, 503] + + +@pytest.mark.django_db +class TestErrorPageRendering: + """Test that error pages render correctly""" + + @patch('config.middleware.connection.ensure_connection') + def test_database_error_page_contains_helpful_message(self, mock_ensure_connection): + """Test that error page has user-friendly content""" + mock_ensure_connection.side_effect = OperationalError("Connection failed") + + factory = RequestFactory() + request = factory.get("/browse") + get_response = Mock() + middleware = DatabaseHealthCheckMiddleware(get_response) + + with patch('time.sleep'): + response = middleware(request) + + # Check response content + assert response.status_code == 503 + content = response.content.decode('utf-8') + assert "We'll be right back!" in content + assert "database" in content.lower() + assert "Retry Now" in content + + @patch('config.middleware.connection.ensure_connection') + def test_database_error_page_has_retry_button(self, mock_ensure_connection): + """Test that error page includes retry functionality""" + mock_ensure_connection.side_effect = OperationalError("Connection failed") + + factory = RequestFactory() + request = factory.get("/") + get_response = Mock() + middleware = DatabaseHealthCheckMiddleware(get_response) + + with patch('time.sleep'): + response = middleware(request) + + content = response.content.decode('utf-8') + assert "location.reload()" in content # Has retry JavaScript + assert "Retry Now" in content # Has retry button + + +@pytest.mark.django_db +class TestDatabaseResilienceIntegration: + """Integration tests for database resilience""" + + def test_middleware_does_not_interfere_with_normal_requests(self, client, test_user): + """Test that middleware doesn't slow down or break normal requests""" + # Make several requests to ensure middleware works normally + response1 = client.get("/") + assert response1.status_code == 200 + + response2 = client.get("/browse/") + assert response2.status_code == 200 + + # Login should work normally + client.login(username="test@example.com", password="testpass123") + response3 = client.get("/dashboard/") + assert response3.status_code == 200 + + def test_middleware_processes_requests_in_correct_order(self): + """Test that middleware is positioned correctly in middleware stack""" + middleware_classes = settings.MIDDLEWARE + + # Find the index of our middleware + db_middleware_index = None + for i, middleware in enumerate(middleware_classes): + if 'DatabaseHealthCheckMiddleware' in middleware: + db_middleware_index = i + break + + assert db_middleware_index is not None, "DatabaseHealthCheckMiddleware not found in MIDDLEWARE" + + # Should be after SecurityMiddleware and WhiteNoiseMiddleware + # but before SessionMiddleware (so we check DB before loading sessions) + security_index = next(i for i, m in enumerate(middleware_classes) if 'SecurityMiddleware' in m) + session_index = next(i for i, m in enumerate(middleware_classes) if 'SessionMiddleware' in m) + + assert db_middleware_index > security_index + assert db_middleware_index < session_index + + +@pytest.mark.django_db +class TestLocalDatabaseTesting: + """Tests for local development database testing""" + + def test_can_manually_test_database_connection(self): + """ + Demonstrates how to manually test database connection. + + To test locally: + 1. Stop your database: brew services stop postgresql (or similar) + 2. Try to access the app + 3. Should see friendly error page + 4. Restart database: brew services start postgresql + 5. Retry button should work + """ + from django.db import connection + + # Verify connection works + connection.ensure_connection() + assert connection.is_usable() + + # You can manually disconnect to test: + # connection.close() + # Then try to make a request + + def test_database_connection_info(self): + """Display database connection info for debugging""" + db_config = settings.DATABASES['default'] + + # These settings should be present + assert 'NAME' in db_config or 'default' in str(db_config) + assert 'ENGINE' in db_config or 'default' in str(db_config) + + print("\nDatabase Configuration:") + print(f" Retry Attempts: {settings.DATABASE_RETRY_ATTEMPTS}") + print(f" Retry Delays: {settings.DATABASE_RETRY_DELAYS}") + print(f" Connection Max Age: {db_config.get('CONN_MAX_AGE', 'Not set')}") + print(f" Health Checks: {db_config.get('CONN_HEALTH_CHECKS', 'Not set')}") diff --git a/test_b2_connection_fixed.py b/test_b2_connection_fixed.py new file mode 100644 index 0000000..4919123 --- /dev/null +++ b/test_b2_connection_fixed.py @@ -0,0 +1,43 @@ +# test_b2_connection_fixed.py +import os +from dotenv import load_dotenv +import certifi + +# Load .env file +load_dotenv() # make sure you have python-dotenv installed + +# Force Python SSL and requests to use certifi CA bundle +os.environ["SSL_CERT_FILE"] = certifi.where() +os.environ["REQUESTS_CA_BUNDLE"] = certifi.where() + +import requests +from b2sdk.v2 import InMemoryAccountInfo, B2Api + +session = requests.Session() +session.verify = certifi.where() + + +# B2 credentials from .env +B2_KEY_ID = os.getenv("B2_KEY_ID") +B2_APPLICATION_KEY = os.getenv("B2_APPLICATION_KEY") +B2_BUCKET_NAME = os.getenv("B2_BUCKET_NAME") +B2_BUCKET_ID = os.getenv("B2_BUCKET_ID") + +print("Testing B2 Configuration...") +print("B2_KEY_ID:", B2_KEY_ID) +print("B2_BUCKET_NAME:", B2_BUCKET_NAME) +print("B2_BUCKET_ID:", B2_BUCKET_ID) +print("B2_APPLICATION_KEY:", "*"*24) + +try: + info = InMemoryAccountInfo() + b2_api = B2Api(info) + b2_api.authorize_account("production", B2_KEY_ID, B2_APPLICATION_KEY) + print("✅ Successfully authorized with B2!") + + bucket = b2_api.get_bucket_by_name(B2_BUCKET_NAME) + print(f"✅ Found bucket: {bucket.name}") + print(f" Bucket ID: {bucket.id_}") + print(f" Bucket Type: {bucket.type_}") +except Exception as e: + print("❌ B2 Connection Error:", e) \ No newline at end of file