|
3 | 3 | import os |
4 | 4 | import re |
5 | 5 | import zipfile |
6 | | -from unittest.mock import MagicMock, mock_open, patch |
| 6 | +from collections.abc import Iterator |
| 7 | +from unittest.mock import MagicMock, PropertyMock, mock_open, patch |
7 | 8 |
|
8 | 9 | import pytest |
9 | 10 | import requests |
| 11 | +import stamina |
10 | 12 |
|
11 | 13 | from ddev.cli.size.utils.common_funcs import ( |
| 14 | + WHEEL_REQUEST_ATTEMPTS, |
| 15 | + WHEEL_REQUEST_TIMEOUT_SECONDS, |
12 | 16 | _matches_gitignore, |
13 | 17 | check_python_version, |
14 | 18 | compress, |
15 | 19 | convert_to_human_readable_size, |
| 20 | + download_wheel, |
16 | 21 | export_format, |
17 | 22 | extract_version_from_about_py, |
18 | 23 | format_modules, |
|
26 | 31 | is_correct_dependency, |
27 | 32 | is_valid_integration_file, |
28 | 33 | request_wheel, |
| 34 | + retry_wheel_request, |
29 | 35 | save_csv, |
30 | 36 | save_json, |
31 | 37 | save_markdown, |
@@ -191,7 +197,11 @@ def test_get_dependencies_sizes(): |
191 | 197 | ) |
192 | 198 |
|
193 | 199 | # The storage tier placeholder must be resolved against the tier passed in, not left as-is. |
194 | | - mock_get.assert_called_once_with("https://example.com/dev/dependency1/dependency1-1.1.1-.whl", stream=True) |
| 200 | + mock_get.assert_called_once_with( |
| 201 | + "https://example.com/dev/dependency1/dependency1-1.1.1-.whl", |
| 202 | + timeout=WHEEL_REQUEST_TIMEOUT_SECONDS, |
| 203 | + stream=True, |
| 204 | + ) |
195 | 205 |
|
196 | 206 | assert file_data == [ |
197 | 207 | { |
@@ -746,4 +756,104 @@ def test_request_wheel_uses_head_when_requested(): |
746 | 756 | with patch("requests.head", return_value=found) as mock_head: |
747 | 757 | assert request_wheel(MagicMock(), PLACEHOLDER_URL, "stable", head=True) is found |
748 | 758 |
|
749 | | - mock_head.assert_called_once_with("https://example.com/stable/built/dep1/dep1-1.1.1-.whl") |
| 759 | + mock_head.assert_called_once_with( |
| 760 | + "https://example.com/stable/built/dep1/dep1-1.1.1-.whl", timeout=WHEEL_REQUEST_TIMEOUT_SECONDS |
| 761 | + ) |
| 762 | + |
| 763 | + |
| 764 | +@pytest.fixture |
| 765 | +def instant_retries() -> Iterator[None]: |
| 766 | + """Skips the backoff between wheel retries while keeping the configured number of attempts.""" |
| 767 | + with stamina.set_testing(True, attempts=100, cap=True): |
| 768 | + yield |
| 769 | + |
| 770 | + |
| 771 | +def make_streamed_wheel_response(content): |
| 772 | + response = make_wheel_response(200) |
| 773 | + response.__enter__.return_value = response |
| 774 | + type(response).content = PropertyMock(side_effect=[content]) |
| 775 | + return response |
| 776 | + |
| 777 | + |
| 778 | +@pytest.mark.parametrize( |
| 779 | + "make_failure", |
| 780 | + [ |
| 781 | + pytest.param(lambda: make_wheel_response(502), id="bad gateway"), |
| 782 | + pytest.param(lambda: make_wheel_response(503), id="service unavailable"), |
| 783 | + pytest.param(lambda: make_wheel_response(504), id="gateway timeout"), |
| 784 | + pytest.param(lambda: requests.exceptions.ConnectionError("Connection reset"), id="connection reset"), |
| 785 | + pytest.param(lambda: requests.exceptions.ReadTimeout("Read timed out"), id="read timeout"), |
| 786 | + pytest.param( |
| 787 | + lambda: make_streamed_wheel_response(requests.exceptions.ChunkedEncodingError("Connection broken")), |
| 788 | + id="body cut short", |
| 789 | + ), |
| 790 | + pytest.param( |
| 791 | + lambda: make_streamed_wheel_response(requests.exceptions.ConnectionError("Read timed out")), |
| 792 | + id="body read timeout", |
| 793 | + ), |
| 794 | + ], |
| 795 | +) |
| 796 | +def test_download_wheel_retries_transient_errors(instant_retries, make_failure): |
| 797 | + with patch("requests.get", side_effect=[make_failure(), make_streamed_wheel_response(b"wheel")]) as mock_get: |
| 798 | + assert download_wheel(MagicMock(), PLACEHOLDER_URL, "dev") == b"wheel" |
| 799 | + |
| 800 | + assert mock_get.call_count == 2 |
| 801 | + |
| 802 | + |
| 803 | +@pytest.mark.parametrize( |
| 804 | + "status_code, expected_requests", |
| 805 | + [ |
| 806 | + pytest.param(500, 1, id="server error"), |
| 807 | + pytest.param(429, 1, id="too many requests"), |
| 808 | + pytest.param(404, 2, id="not found on every tier"), |
| 809 | + ], |
| 810 | +) |
| 811 | +def test_download_wheel_does_not_retry_other_errors(instant_retries, status_code, expected_requests): |
| 812 | + responses = [make_wheel_response(status_code) for _ in range(WHEEL_REQUEST_ATTEMPTS)] |
| 813 | + |
| 814 | + with patch("requests.get", side_effect=responses) as mock_get: |
| 815 | + with pytest.raises(requests.HTTPError, match=str(status_code)): |
| 816 | + download_wheel(MagicMock(), PLACEHOLDER_URL, "dev") |
| 817 | + |
| 818 | + assert mock_get.call_count == expected_requests |
| 819 | + |
| 820 | + |
| 821 | +def make_failing_responses(kinds): |
| 822 | + return [ |
| 823 | + make_wheel_response(504) |
| 824 | + if kind == "gateway error" |
| 825 | + else make_streamed_wheel_response(requests.exceptions.ChunkedEncodingError("Connection broken")) |
| 826 | + for kind in kinds |
| 827 | + ] |
| 828 | + |
| 829 | + |
| 830 | +@pytest.mark.parametrize( |
| 831 | + "kinds", |
| 832 | + [ |
| 833 | + pytest.param(["gateway error"] * (WHEEL_REQUEST_ATTEMPTS + 2), id="gateway errors"), |
| 834 | + pytest.param(["body cut short"] * (WHEEL_REQUEST_ATTEMPTS + 2), id="bodies cut short"), |
| 835 | + pytest.param( |
| 836 | + ["gateway error"] * (WHEEL_REQUEST_ATTEMPTS - 1) + ["body cut short"] * 3, |
| 837 | + id="gateway errors then a body cut short", |
| 838 | + ), |
| 839 | + ], |
| 840 | +) |
| 841 | +def test_download_wheel_shares_one_attempt_budget_and_names_the_wheel(instant_retries, kinds): |
| 842 | + with patch("requests.get", side_effect=make_failing_responses(kinds)) as mock_get: |
| 843 | + with pytest.raises(requests.RequestException) as excinfo: |
| 844 | + download_wheel(MagicMock(), PLACEHOLDER_URL, "dev") |
| 845 | + |
| 846 | + assert mock_get.call_count == WHEEL_REQUEST_ATTEMPTS |
| 847 | + assert any("dep1-1.1.1-.whl" in note for note in excinfo.value.__notes__) |
| 848 | + |
| 849 | + |
| 850 | +def test_retry_wheel_request_retries_a_head_request(instant_retries): |
| 851 | + found = make_wheel_response(200) |
| 852 | + |
| 853 | + with patch("requests.head", side_effect=[make_wheel_response(504), found]) as mock_head: |
| 854 | + response = retry_wheel_request( |
| 855 | + lambda: request_wheel(MagicMock(), PLACEHOLDER_URL, "stable", head=True), PLACEHOLDER_URL |
| 856 | + ) |
| 857 | + |
| 858 | + assert response is found |
| 859 | + assert mock_head.call_count == 2 |
0 commit comments