Skip to content

Fix wrong asset 404 error #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Salamandar merged 2 commits into from
Nov 4, 2024
Merged

Fix wrong asset 404 error #178

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3140d58
Calculate full_url only once
Salamandar Nov 4, 2024
b049157
Extract base tag href for resolving asset URLs
fflorent Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions lib/curl_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import tempfile
import pycurl
from bs4 import BeautifulSoup
from urllib.parse import urlencode
from urllib.parse import urlencode, urljoin
from io import BytesIO

DOMAIN = os.environ["DOMAIN"]
Expand Down Expand Up @@ -141,12 +141,13 @@ def test(
else:
cookies = None

full_url = base_url + path
code = None
retried = 0
while code is None or code in {502, 503, 504}:
time.sleep(retried * 5)
code, content, effective_url = curl(
base_url + path, post=post, use_cookies=cookies
full_url, post=post, use_cookies=cookies
)
retried += 1
if retried > 3:
Expand All @@ -164,6 +165,9 @@ def test(
content = content.get_text().strip() if content else ""
content = re.sub(r"[\t\n\s]{3,}", "\n\n", content)

base_tag = html.find('base')
base = base_tag['href'] if base_tag else ''

errors = []
if expect_effective_url is None and "/yunohost/sso" in effective_url:
errors.append(
Expand Down Expand Up @@ -229,18 +233,19 @@ def test(
elif asset.startswith(f"{domain}/"):
asset = asset.replace(f"{domain}/", "")
if not asset.startswith("/"):
asset = "/" + asset
asset = urljoin(base + "/", asset)
resolved_asset_url = urljoin(f"https://{domain}", asset)
asset_code, _, effective_asset_url = curl(
f"https://{domain}{asset}", use_cookies=cookies
resolved_asset_url, use_cookies=cookies
)
if asset_code != 200:
errors.append(
f"Asset https://{domain}{asset} (automatically derived from the page's html) answered with code {asset_code}, expected 200? Effective url: {effective_asset_url}"
f"Asset {resolved_asset_url} (automatically derived from the page's html) answered with code {asset_code}, expected 200? Effective url: {effective_asset_url}"
)
assets.append((domain + asset, asset_code))
assets.append((resolved_asset_url, asset_code))

return {
"url": f"{base_url}{path}",
"url": full_url,
"effective_url": effective_url,
"code": code,
"title": title,
Expand Down