From 2831c75e6240962e47f78d4f92e959f06ca98434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:23:17 +0200 Subject: [PATCH 01/15] Update certbot-new.py Currently the API_KEY only version is not working properly. You must add EAB_KID AND EAB_HMAC_KEY ACME Certificate Authority Settings (New) Variable | Default | Options | Description ACME_SSL_CA_PROVIDER | "letsencrypt" | "letsencrypt", "zerossl" | Choose certificate authority ACME_ZEROSSL_API_KEY | "" | API key string | ZeroSSL API key for EAB setup ZeroSSL EAB Credentials (New - Alternative to API Key) Variable | Default | Options | Description ACME_ZEROSSL_EAB_KID | "" | EAB Kid string | Manual ZeroSSL EAB Kid ACME_ZEROSSL_EAB_HMAC_KEY | "" | EAB HMAC string | Manual ZeroSSL EAB HMAC Key HTTP Challenge Validation Settings (New) Variable | Default | Options | Description ACME_SKIP_HTTP_DNS_CHECK | "no" | "yes", "no" | Skip DNS A/AAAA record validation ACME_SKIP_CAA_CHECK | "no" | "yes", "no" | Skip CAA record validation ACME_HTTP_STRICT_IP_CHECK | "no" | "yes", "no" | Fail on server IP mismatches ACME_DNS_RETRY_COUNT | "2" | Number (0-10) | DNS lookup retry attempts For multisite mode (MULTISITE="yes"), prefix any variable with {SERVER}_: example.com_ACME_SSL_CA_PROVIDER="zerossl" example.com_ACME_ZEROSSL_API_KEY="your_api_key" example.com_ACME_ZEROSSL_EAB_KID="XYZ" example.com_ACME_ZEROSSL_EAB_HMAC_KEY="ABCABC" Features added: ACME_SKIP_HTTP_DNS_CHECK (default no) - DNS A/AAAA record validation ACME_SKIP_CAA_CHECK (default no) - CAA record validation (dig required) ACME_HTTP_STRICT_IP_CHECK (default no) - Fail on server IP mismatches - I really recommend to set it to yes to verify misconfigurations The ACME_SKIP_HTTP_DNS_CHECK = no means your instance will do a DNS lookup when using HTTP verification. If your DNS record is missing it will hard fail. So you don't loose one request using ACME because of a failed DNS record entry --- .../core/letsencrypt/jobs/certbot-new.py | 1059 ++++++++++++++--- 1 file changed, 902 insertions(+), 157 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index f49e841b2c..d28ebdbb98 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -11,13 +11,16 @@ from pathlib import Path from re import MULTILINE, match, search from select import select +from shutil import which +from socket import getaddrinfo, gaierror, AF_INET, AF_INET6 from subprocess import DEVNULL, PIPE, STDOUT, Popen, run from sys import exit as sys_exit, path as sys_path from time import sleep from traceback import format_exc from typing import Dict, Literal, Type, Union -for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) + for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -67,13 +70,21 @@ PSL_URL = "https://publicsuffix.org/list/public_suffix_list.dat" PSL_STATIC_FILE = "public_suffix_list.dat" +# ZeroSSL Configuration +ZEROSSL_ACME_SERVER = "https://acme.zerossl.com/v2/DV90" +ZEROSSL_STAGING_SERVER = "https://acme.zerossl.com/v2/DV90" # ZeroSSL doesn't have staging +LETSENCRYPT_ACME_SERVER = "https://acme-v02.api.letsencrypt.org/directory" +LETSENCRYPT_STAGING_SERVER = "https://acme-staging-v02.api.letsencrypt.org/directory" + def load_public_suffix_list(job): + # Load and cache the public suffix list for domain validation job_cache = job.get_cache(PSL_STATIC_FILE, with_info=True, with_data=True) if ( isinstance(job_cache, dict) and job_cache.get("last_update") - and job_cache["last_update"] < (datetime.now().astimezone() - timedelta(days=1)).timestamp() + and job_cache["last_update"] < (datetime.now().astimezone() - + timedelta(days=1)).timestamp() ): return job_cache["data"].decode("utf-8").splitlines() @@ -101,41 +112,520 @@ def parse_psl(psl_lines): for line in psl_lines: line = line.strip() if not line or line.startswith("//"): - continue # Ignore empty lines and comments + continue if line.startswith("!"): - exceptions.add(line[1:]) # Exception rule + exceptions.add(line[1:]) continue - rules.add(line) # Normal or wildcard rule + rules.add(line) return {"rules": rules, "exceptions": exceptions} def is_domain_blacklisted(domain, psl): - # Returns True if the domain is forbidden by PSL rules + # Check if domain is forbidden by PSL rules domain = domain.lower().strip(".") labels = domain.split(".") for i in range(len(labels)): candidate = ".".join(labels[i:]) - # Allow if candidate is an exception if candidate in psl["exceptions"]: return False - # Block if candidate matches a PSL rule if candidate in psl["rules"]: if i == 0: - return True # Block exact match + return True if i == 0 and domain.startswith("*."): - return True # Block wildcard for the rule itself + return True if i == 0 or (i == 1 and labels[0] == "*"): - return True # Block *.domain.tld + return True if len(labels[i:]) == len(labels): - return True # Block domain.tld - # Allow subdomains - # Block if candidate matches a PSL wildcard rule + return True if f"*.{candidate}" in psl["rules"]: if len(labels[i:]) == 2: - return True # Block foo.bar and *.foo.bar + return True return False +def get_certificate_authority_config(ca_provider, staging=False): + # Get ACME server configuration for the specified CA provider + if ca_provider.lower() == "zerossl": + return { + "server": ZEROSSL_STAGING_SERVER if staging else ZEROSSL_ACME_SERVER, + "name": "ZeroSSL" + } + else: # Default to Let's Encrypt + return { + "server": LETSENCRYPT_STAGING_SERVER if staging else LETSENCRYPT_ACME_SERVER, + "name": "Let's Encrypt" + } + + +def setup_zerossl_eab_credentials(email, api_key=None): + # Setup External Account Binding (EAB) credentials for ZeroSSL + LOGGER.info(f"Setting up ZeroSSL EAB credentials for email: {email}") + + if not api_key: + LOGGER.error("❌ ZeroSSL API key not provided") + LOGGER.warning("ZeroSSL API key not provided, attempting registration with email") + return None, None + + LOGGER.info("Making request to ZeroSSL API for EAB credentials") + + # Try the correct ZeroSSL API endpoint + try: + # The correct endpoint for ZeroSSL EAB credentials + response = get( + "https://api.zerossl.com/acme/eab-credentials", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30 + ) + LOGGER.info(f"ZeroSSL API response status: {response.status_code}") + + if response.status_code == 200: + response.raise_for_status() + eab_data = response.json() + LOGGER.info(f"ZeroSSL API response data: {eab_data}") + + # ZeroSSL typically returns eab_kid and eab_hmac_key directly + if "eab_kid" in eab_data and "eab_hmac_key" in eab_data: + eab_kid = eab_data.get("eab_kid") + eab_hmac_key = eab_data.get("eab_hmac_key") + LOGGER.info(f"✓ Successfully obtained EAB credentials from ZeroSSL") + LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") + LOGGER.info(f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}...") + return eab_kid, eab_hmac_key + else: + LOGGER.error(f"❌ Invalid ZeroSSL API response format: {eab_data}") + return None, None + else: + # Try alternative endpoint if first one fails + LOGGER.warning(f"Primary endpoint failed with {response.status_code}, trying alternative") + response_text = response.text + LOGGER.info(f"Primary endpoint response: {response_text}") + + # Try alternative endpoint with email parameter + response = get( + "https://api.zerossl.com/acme/eab-credentials-email", + params={"email": email}, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=30 + ) + LOGGER.info(f"Alternative ZeroSSL API response status: {response.status_code}") + response.raise_for_status() + eab_data = response.json() + + LOGGER.info(f"Alternative ZeroSSL API response data: {eab_data}") + + if eab_data.get("success"): + eab_kid = eab_data.get("eab_kid") + eab_hmac_key = eab_data.get("eab_hmac_key") + LOGGER.info(f"✓ Successfully obtained EAB credentials from ZeroSSL (alternative endpoint)") + LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") + LOGGER.info(f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}...") + return eab_kid, eab_hmac_key + else: + LOGGER.error(f"❌ ZeroSSL EAB registration failed: {eab_data}") + return None, None + + except BaseException as e: + LOGGER.debug(format_exc()) + LOGGER.error(f"❌ Error setting up ZeroSSL EAB credentials: {e}") + + # Additional troubleshooting info + LOGGER.error("Troubleshooting steps:") + LOGGER.error("1. Verify your ZeroSSL API key is valid") + LOGGER.error("2. Check your ZeroSSL account has ACME access enabled") + LOGGER.error("3. Ensure the API key has the correct permissions") + LOGGER.error("4. Try regenerating your ZeroSSL API key") + + return None, None + + +def get_caa_records(domain): + # Get CAA records for a domain using dig command + + # Check if dig command is available + if not which("dig"): + LOGGER.info("dig command not available for CAA record checking") + return None + + try: + # Use dig to query CAA records + LOGGER.info(f"Querying CAA records for domain: {domain}") + result = run( + ["dig", "+short", domain, "CAA"], + capture_output=True, + text=True, + timeout=10 + ) + + if result.returncode == 0 and result.stdout.strip(): + LOGGER.info(f"Found CAA records for domain {domain}") + caa_records = [] + for line in result.stdout.strip().split('\n'): + line = line.strip() + if line: + # CAA record format: flags tag "value" + # Example: 0 issue "letsencrypt.org" + parts = line.split(' ', 2) + if len(parts) >= 3: + flags = parts[0] + tag = parts[1] + value = parts[2].strip('"') + caa_records.append({ + 'flags': flags, + 'tag': tag, + 'value': value + }) + LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain {domain}") + return caa_records + else: + LOGGER.info(f"No CAA records found for domain {domain} (dig return code: {result.returncode})") + return [] + + except BaseException as e: + LOGGER.info(f"Error querying CAA records for {domain}: {e}") + return None + + +def check_caa_authorization(domain, ca_provider, is_wildcard=False): + # Check if the CA provider is authorized by CAA records + + LOGGER.info(f"Checking CAA authorization for domain: {domain}, CA: {ca_provider}, wildcard: {is_wildcard}") + + # Map CA providers to their CAA identifiers + ca_identifiers = { + "letsencrypt": ["letsencrypt.org"], + "zerossl": ["sectigo.com", "zerossl.com"] # ZeroSSL uses Sectigo + } + + allowed_identifiers = ca_identifiers.get(ca_provider.lower(), []) + if not allowed_identifiers: + LOGGER.warning(f"Unknown CA provider for CAA check: {ca_provider}") + return True # Allow unknown providers (conservative approach) + + # Check CAA records for the domain and parent domains + check_domain = domain.lstrip("*.") + domain_parts = check_domain.split(".") + LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") + + for i in range(len(domain_parts)): + current_domain = ".".join(domain_parts[i:]) + LOGGER.info(f"Checking CAA records for: {current_domain}") + caa_records = get_caa_records(current_domain) + + if caa_records is None: + # dig not available, skip CAA check + LOGGER.info("CAA record checking skipped (dig command not available)") + return True + + if caa_records: + LOGGER.info(f"Found CAA records for {current_domain}") + + # Check relevant CAA records + issue_records = [] + issuewild_records = [] + + for record in caa_records: + if record['tag'] == 'issue': + issue_records.append(record['value']) + elif record['tag'] == 'issuewild': + issuewild_records.append(record['value']) + + # Log found records + if issue_records: + LOGGER.info(f"CAA issue records: {', '.join(issue_records)}") + if issuewild_records: + LOGGER.info(f"CAA issuewild records: {', '.join(issuewild_records)}") + + # Check authorization based on certificate type + if is_wildcard: + # For wildcard certificates, check issuewild first, then fall back to issue + check_records = issuewild_records if issuewild_records else issue_records + record_type = "issuewild" if issuewild_records else "issue" + else: + # For regular certificates, check issue records + check_records = issue_records + record_type = "issue" + + LOGGER.info(f"Using CAA {record_type} records for authorization check") + + if not check_records: + LOGGER.info(f"No relevant CAA {record_type} records found for {current_domain}") + continue + + # Check if any of our CA identifiers are authorized + authorized = False + LOGGER.info(f"Checking authorization for CA identifiers: {', '.join(allowed_identifiers)}") + for identifier in allowed_identifiers: + for record in check_records: + # Handle explicit deny (empty value or semicolon) + if record == ";" or record.strip() == "": + LOGGER.warning(f"CAA {record_type} record explicitly denies all CAs") + return False + + # Check for CA authorization + if identifier in record: + authorized = True + LOGGER.info(f"✓ CA {ca_provider} ({identifier}) authorized by CAA {record_type} record") + break + if authorized: + break + + if not authorized: + LOGGER.error(f"❌ CA {ca_provider} is NOT authorized by CAA {record_type} records") + LOGGER.error(f"Domain {current_domain} CAA {record_type} allows: {', '.join(check_records)}") + LOGGER.error(f"But {ca_provider} uses: {', '.join(allowed_identifiers)}") + return False + + # If we found CAA records and we're authorized, we can stop checking parent domains + LOGGER.info(f"✓ CAA authorization successful for {domain}") + return True + + # No CAA records found in the entire chain + LOGGER.info(f"No CAA records found for {check_domain} or parent domains - any CA allowed") + return True + + +def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", is_wildcard=False): + # Validate that all domains have valid A/AAAA records and CAA authorization for HTTP challenge + LOGGER.info(f"Validating {len(domains_list)} domains for HTTP challenge: {', '.join(domains_list)}") + invalid_domains = [] + caa_blocked_domains = [] + + # Check if CAA validation should be skipped + skip_caa_check = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" + + # Get external IPs once for all domain checks + external_ips = get_external_ip() + if external_ips: + if external_ips.get("ipv4"): + LOGGER.info(f"Server external IPv4 address: {external_ips['ipv4']}") + if external_ips.get("ipv6"): + LOGGER.info(f"Server external IPv6 address: {external_ips['ipv6']}") + else: + LOGGER.warning("Could not determine server external IP - skipping IP match validation") + + for domain in domains_list: + # Check DNS A/AAAA records with retry mechanism + if not check_domain_a_record(domain, external_ips): + invalid_domains.append(domain) + continue + + # Check CAA authorization + if not skip_caa_check: + if not check_caa_authorization(domain, ca_provider, is_wildcard): + caa_blocked_domains.append(domain) + else: + LOGGER.info(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") + + # Report results + if invalid_domains: + LOGGER.error(f"The following domains do not have valid A/AAAA records and cannot be used " + f"for HTTP challenge: {', '.join(invalid_domains)}") + LOGGER.error("Please ensure domains resolve to this server before requesting certificates") + return False + + if caa_blocked_domains: + LOGGER.error(f"The following domains have CAA records that block {ca_provider}: " + f"{', '.join(caa_blocked_domains)}") + LOGGER.error("Please update CAA records to authorize the certificate authority or use a different CA") + LOGGER.info("You can skip CAA checking by setting ACME_SKIP_CAA_CHECK=yes") + return False + + LOGGER.info(f"All domains have valid DNS records and CAA authorization for HTTP challenge: {', '.join(domains_list)}") + return True + + +def get_external_ip(): + # Get the external/public IP addresses of this server (both IPv4 and IPv6) + LOGGER.info("Getting external IP addresses for server") + ipv4_services = [ + "https://ipv4.icanhazip.com", + "https://api.ipify.org", + "https://checkip.amazonaws.com", + "https://ipv4.jsonip.com" + ] + + ipv6_services = [ + "https://ipv6.icanhazip.com", + "https://api6.ipify.org", + "https://ipv6.jsonip.com" + ] + + external_ips = {"ipv4": None, "ipv6": None} + + # Try to get IPv4 address + LOGGER.info("Attempting to get external IPv4 address") + for service in ipv4_services: + try: + if "jsonip.com" in service: + # This service returns JSON format + response = get(service, timeout=5) + response.raise_for_status() + data = response.json() + ip = data.get("ip", "").strip() + else: + # These services return plain text IP + response = get(service, timeout=5) + response.raise_for_status() + ip = response.text.strip() + + # Basic IPv4 validation + if ip and "." in ip and len(ip.split(".")) == 4: + try: + # Validate it's a proper IPv4 address + getaddrinfo(ip, None, AF_INET) + external_ips["ipv4"] = ip + LOGGER.info(f"Successfully obtained external IPv4 address: {ip}") + break + except gaierror: + continue + except BaseException as e: + LOGGER.info(f"Failed to get IPv4 address from {service}: {e}") + continue + + # Try to get IPv6 address + LOGGER.info("Attempting to get external IPv6 address") + for service in ipv6_services: + try: + if "jsonip.com" in service: + response = get(service, timeout=5) + response.raise_for_status() + data = response.json() + ip = data.get("ip", "").strip() + else: + response = get(service, timeout=5) + response.raise_for_status() + ip = response.text.strip() + + # Basic IPv6 validation + if ip and ":" in ip: + try: + # Validate it's a proper IPv6 address + getaddrinfo(ip, None, AF_INET6) + external_ips["ipv6"] = ip + LOGGER.info(f"Successfully obtained external IPv6 address: {ip}") + break + except gaierror: + continue + except BaseException as e: + LOGGER.info(f"Failed to get IPv6 address from {service}: {e}") + continue + + if not external_ips["ipv4"] and not external_ips["ipv6"]: + LOGGER.warning("Could not determine external IP address (IPv4 or IPv6) from any service") + return None + + LOGGER.info(f"External IP detection completed - IPv4: {external_ips['ipv4'] or 'not found'}, IPv6: {external_ips['ipv6'] or 'not found'}") + return external_ips + + +def check_domain_a_record(domain, external_ips=None): + # Check if domain has valid A/AAAA records for HTTP challenge + LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") + try: + # Remove wildcard prefix if present + check_domain = domain.lstrip("*.") + + # Attempt to resolve the domain to IP addresses + result = getaddrinfo(check_domain, None) + if result: + ipv4_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET] + ipv6_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET6] + + if not ipv4_addresses and not ipv6_addresses: + LOGGER.warning(f"Domain {check_domain} has no A or AAAA records") + return False + + # Log found addresses + if ipv4_addresses: + LOGGER.info(f"Domain {check_domain} IPv4 A records: {', '.join(ipv4_addresses[:3])}") + if ipv6_addresses: + LOGGER.info(f"Domain {check_domain} IPv6 AAAA records: {', '.join(ipv6_addresses[:3])}") + + # Check if any record matches the external IPs + if external_ips: + ipv4_match = False + ipv6_match = False + + # Check IPv4 match + if external_ips.get("ipv4") and ipv4_addresses: + if external_ips["ipv4"] in ipv4_addresses: + LOGGER.info(f"✓ Domain {check_domain} IPv4 A record matches server external IP ({external_ips['ipv4']})") + ipv4_match = True + else: + LOGGER.warning(f"⚠ Domain {check_domain} IPv4 A record does not match server external IP") + LOGGER.warning(f" Domain IPv4: {', '.join(ipv4_addresses)}") + LOGGER.warning(f" Server IPv4: {external_ips['ipv4']}") + + # Check IPv6 match + if external_ips.get("ipv6") and ipv6_addresses: + if external_ips["ipv6"] in ipv6_addresses: + LOGGER.info(f"✓ Domain {check_domain} IPv6 AAAA record matches server external IP ({external_ips['ipv6']})") + ipv6_match = True + else: + LOGGER.warning(f"⚠ Domain {check_domain} IPv6 AAAA record does not match server external IP") + LOGGER.warning(f" Domain IPv6: {', '.join(ipv6_addresses)}") + LOGGER.warning(f" Server IPv6: {external_ips['ipv6']}") + + # Determine if we have any matching records + has_any_match = ipv4_match or ipv6_match + has_external_ip = external_ips.get("ipv4") or external_ips.get("ipv6") + + if has_external_ip and not has_any_match: + LOGGER.warning(f"⚠ Domain {check_domain} records do not match any server external IP") + LOGGER.warning(f" HTTP challenge may fail - ensure domain points to this server") + + # Check if we should treat this as an error + strict_ip_check = getenv("ACME_HTTP_STRICT_IP_CHECK", "no") == "yes" + if strict_ip_check: + LOGGER.error(f"Strict IP check enabled - rejecting certificate request for {check_domain}") + return False + + LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") + return True + else: + LOGGER.info(f"Domain {check_domain} validation failed - no DNS resolution") + LOGGER.warning(f"Domain {check_domain} does not resolve") + return False + + except gaierror as e: + LOGGER.info(f"Domain {check_domain} DNS resolution failed (gaierror): {e}") + LOGGER.warning(f"DNS resolution failed for domain {check_domain}: {e}") + return False + except BaseException as e: + LOGGER.info(format_exc()) + LOGGER.error(f"Error checking DNS records for domain {check_domain}: {e}") + return False + + +def validate_domains_for_http_challenge(domains_list): + # Validate that all domains have valid A/AAAA records for HTTP challenge + LOGGER.info(f"Validating {len(domains_list)} domains for HTTP challenge: {', '.join(domains_list)}") + invalid_domains = [] + + # Get external IPs once for all domain checks + external_ips = get_external_ip() + if external_ips: + if external_ips.get("ipv4"): + LOGGER.info(f"Server external IPv4 address: {external_ips['ipv4']}") + if external_ips.get("ipv6"): + LOGGER.info(f"Server external IPv6 address: {external_ips['ipv6']}") + else: + LOGGER.warning("Could not determine server external IP - skipping IP match validation") + + for domain in domains_list: + if not check_domain_a_record(domain, external_ips): + invalid_domains.append(domain) + + if invalid_domains: + LOGGER.error(f"The following domains do not have valid A/AAAA records and cannot be used " + f"for HTTP challenge: {', '.join(invalid_domains)}") + LOGGER.error("Please ensure domains resolve to this server before requesting certificates") + return False + + LOGGER.info(f"All domains have valid DNS records for HTTP challenge: {', '.join(domains_list)}") + return True + + def certbot_new_with_retry( challenge_type: Literal["dns", "http"], domains: str, @@ -148,14 +638,17 @@ def certbot_new_with_retry( force: bool = False, cmd_env: Dict[str, str] = None, max_retries: int = 0, + ca_provider: str = "letsencrypt", + api_key: str = None, + server_name: str = None, ) -> int: - """Execute certbot with retry mechanism.""" + # Execute certbot with retry mechanism attempt = 1 - while attempt <= max_retries + 1: # +1 for the initial attempt + while attempt <= max_retries + 1: if attempt > 1: - LOGGER.warning(f"Certificate generation failed, retrying... (attempt {attempt}/{max_retries + 1})") - # Wait before retrying (exponential backoff: 30s, 60s, 120s...) - wait_time = min(30 * (2 ** (attempt - 2)), 300) # Cap at 5 minutes + LOGGER.warning(f"Certificate generation failed, retrying... " + f"(attempt {attempt}/{max_retries + 1})") + wait_time = min(30 * (2 ** (attempt - 2)), 300) LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) @@ -170,6 +663,9 @@ def certbot_new_with_retry( staging, force, cmd_env, + ca_provider, + api_key, + server_name, ) if result == 0: @@ -197,11 +693,16 @@ def certbot_new( staging: bool = False, force: bool = False, cmd_env: Dict[str, str] = None, + ca_provider: str = "letsencrypt", + api_key: str = None, + server_name: str = None, ) -> int: + # Generate new certificate using certbot if isinstance(credentials_path, str): credentials_path = Path(credentials_path) - # * Building the certbot command + ca_config = get_certificate_authority_config(ca_provider, staging) + command = [ CERTBOT_BIN, "certonly", @@ -219,25 +720,70 @@ def certbot_new( "--agree-tos", "--expand", f"--preferred-profile={profile}", + "--server", + ca_config["server"], ] if not cmd_env: cmd_env = {} + # Handle certificate key type based on DNS provider and CA + if challenge_type == "dns" and provider in ("infomaniak", "ionos"): + # Infomaniak and IONOS require RSA certificates with 4096-bit keys + command.extend(["--rsa-key-size", "4096"]) + LOGGER.info(f"Using RSA-4096 for {provider} provider with {domains}") + else: + # Use elliptic curve certificates for all other providers + if ca_provider.lower() == "zerossl": + # Use P-384 elliptic curve for ZeroSSL certificates + command.extend(["--elliptic-curve", "secp384r1"]) + LOGGER.info(f"Using ZeroSSL P-384 curve for {domains}") + else: + # Use P-256 elliptic curve for Let's Encrypt certificates + command.extend(["--elliptic-curve", "secp256r1"]) + LOGGER.info(f"Using Let's Encrypt P-256 curve for {domains}") + + # Handle ZeroSSL EAB credentials + if ca_provider.lower() == "zerossl": + LOGGER.info(f"ZeroSSL detected as CA provider for {domains}") + + # Check for manually provided EAB credentials first + eab_kid_env = getenv("ACME_ZEROSSL_EAB_KID", "") or getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "") + eab_hmac_env = getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") + + if eab_kid_env and eab_hmac_env: + LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials from environment") + command.extend(["--eab-kid", eab_kid_env, "--eab-hmac-key", eab_hmac_env]) + LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") + LOGGER.info(f"EAB Kid: {eab_kid_env[:10]}...") + elif api_key: + LOGGER.info(f"ZeroSSL API key provided, setting up EAB credentials") + eab_kid, eab_hmac = setup_zerossl_eab_credentials(email, api_key) + if eab_kid and eab_hmac: + command.extend(["--eab-kid", eab_kid, "--eab-hmac-key", eab_hmac]) + LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") + LOGGER.info(f"EAB Kid: {eab_kid[:10]}...") + else: + LOGGER.error("❌ Failed to obtain ZeroSSL EAB credentials") + LOGGER.error("Alternative: Set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY environment variables") + LOGGER.warning("Proceeding without EAB - this will likely fail") + else: + LOGGER.error("❌ No ZeroSSL API key provided!") + LOGGER.error("Set ACME_ZEROSSL_API_KEY environment variable") + LOGGER.error("Or set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY directly") + LOGGER.warning("Proceeding without EAB - this will likely fail") + if challenge_type == "dns": - # * Adding DNS challenge hooks command.append("--preferred-challenges=dns") - # * Adding the propagation time to the command if propagation != "default": if not propagation.isdigit(): - LOGGER.warning(f"Invalid propagation time : {propagation}, using provider's default...") + LOGGER.warning(f"Invalid propagation time : {propagation}, " + "using provider's default...") else: command.extend([f"--dns-{provider}-propagation-seconds", propagation]) - # * Adding the credentials to the command if provider == "route53": - # ? Route53 credentials are different from the others, we need to add them to the environment with credentials_path.open("r") as file: for line in file: key, value = line.strip().split("=", 1) @@ -245,19 +791,12 @@ def certbot_new( else: command.extend([f"--dns-{provider}-credentials", credentials_path.as_posix()]) - # * Adding the RSA key size argument like in the infomaniak plugin documentation - if provider in ("infomaniak", "ionos"): - command.extend(["--rsa-key-size", "4096"]) - - # * Adding plugin argument if provider in ("desec", "infomaniak", "ionos", "njalla", "scaleway"): - # ? Desec, Infomaniak, IONOS, Njalla and Scaleway plugins use different arguments command.extend(["--authenticator", f"dns-{provider}"]) else: command.append(f"--dns-{provider}") elif challenge_type == "http": - # * Adding HTTP challenge hooks command.extend( [ "--manual", @@ -269,15 +808,27 @@ def certbot_new( ] ) - if staging: - command.append("--staging") - if force: command.append("--force-renewal") if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": command.append("-v") + LOGGER.info(f"Executing certbot command for {domains}") + # Show command but mask sensitive EAB values for security + safe_command = [] + mask_next = False + for item in command: + if mask_next: + safe_command.append("***MASKED***") + mask_next = False + elif item in ["--eab-kid", "--eab-hmac-key"]: + safe_command.append(item) + mask_next = True + else: + safe_command.append(item) + LOGGER.info(f"Command: {' '.join(safe_command)}") + current_date = datetime.now() process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, env=cmd_env) @@ -290,9 +841,10 @@ def certbot_new( break if datetime.now() - current_date > timedelta(seconds=5): - LOGGER.info( - "⏳ Still generating certificate(s)" + (" (this may take a while depending on the provider)" if challenge_type == "dns" else "") + "..." - ) + challenge_info = (" (this may take a while depending on the provider)" + if challenge_type == "dns" else "") + LOGGER.info(f"⏳ Still generating {ca_config['name']} certificate(s)" + f"{challenge_info}...") current_date = datetime.now() return process.returncode @@ -324,10 +876,12 @@ def certbot_new( if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes": use_letsencrypt = True - if first_server and getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") == "dns": + if first_server and getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", + "http") == "dns": use_letsencrypt_dns = True - domains_server_names[first_server] = getenv(f"{first_server}_SERVER_NAME", first_server).lower() + domains_server_names[first_server] = getenv(f"{first_server}_SERVER_NAME", + first_server).lower() if not use_letsencrypt: LOGGER.info("Let's Encrypt is not activated, skipping generation...") @@ -381,16 +935,19 @@ def certbot_new( JOB = Job(LOGGER, __file__) - # ? Restore data from db cache of certbot-renew job + # Restore data from db cache of certbot-renew job JOB.restore_cache(job_name="certbot-renew") env = { "PATH": getenv("PATH", ""), "PYTHONPATH": getenv("PYTHONPATH", ""), "RELOAD_MIN_TIMEOUT": getenv("RELOAD_MIN_TIMEOUT", "5"), - "DISABLE_CONFIGURATION_TESTING": getenv("DISABLE_CONFIGURATION_TESTING", "no").lower(), + "DISABLE_CONFIGURATION_TESTING": getenv("DISABLE_CONFIGURATION_TESTING", + "no").lower(), } - env["PYTHONPATH"] = env["PYTHONPATH"] + (f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "") + env["PYTHONPATH"] = env["PYTHONPATH"] + (f":{DEPS_PATH}" + if DEPS_PATH not in env["PYTHONPATH"] + else "") if getenv("DATABASE_URI"): env["DATABASE_URI"] = getenv("DATABASE_URI") @@ -418,22 +975,27 @@ def certbot_new( credential_paths = set() generated_domains = set() domains_to_ask = {} - active_cert_names = set() # Track ALL active certificate names, not just processed ones + active_cert_names = set() if proc.returncode != 0: LOGGER.error(f"Error while checking certificates :\n{proc.stdout}") else: certificate_blocks = stdout.split("Certificate Name: ")[1:] for first_server, domains in domains_server_names.items(): - if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": + if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": continue - letsencrypt_challenge = getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") if IS_MULTISITE else getenv("LETS_ENCRYPT_CHALLENGE", "http") + letsencrypt_challenge = (getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", + "http") if IS_MULTISITE + else getenv("LETS_ENCRYPT_CHALLENGE", "http")) original_first_server = deepcopy(first_server) if ( letsencrypt_challenge == "dns" - and (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes" + and (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes" ): wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains((first_server,)) first_server = wildcards[0].lstrip("*.") @@ -452,46 +1014,87 @@ def certbot_new( if not certificate_block: domains_to_ask[first_server] = 1 - LOGGER.warning(f"[{original_first_server}] Certificate block for {first_server} not found, asking new certificate...") + LOGGER.warning(f"[{original_first_server}] Certificate block for " + f"{first_server} not found, asking new certificate...") continue + # Validating the credentials try: - cert_domains = search(r"Domains: (?P.*)\n\s*Expiry Date: (?P.*)\n", certificate_block, MULTILINE) + cert_domains = search(r"Domains: (?P.*)\n\s*Expiry Date: " + r"(?P.*)\n", certificate_block, MULTILINE) except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"[{original_first_server}] Error while parsing certificate block: {e}") + LOGGER.error(f"[{original_first_server}] Error while parsing " + f"certificate block: {e}") continue if not cert_domains: - LOGGER.error(f"[{original_first_server}] Failed to parse domains and expiry date from certificate block.") + LOGGER.error(f"[{original_first_server}] Failed to parse domains " + "and expiry date from certificate block.") continue cert_domains_list = cert_domains.group("domains").strip().split() cert_domains_set = set(cert_domains_list) - desired_domains_set = set(domains) if isinstance(domains, (list, set)) else set(domains.split()) + desired_domains_set = (set(domains) if isinstance(domains, (list, set)) + else set(domains.split())) if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 LOGGER.warning( - f"[{original_first_server}] Domains for {first_server} differ from desired set (existing: {sorted(cert_domains_set)}, desired: {sorted(desired_domains_set)}), asking new certificate..." + f"[{original_first_server}] Domains for {first_server} differ " + f"from desired set (existing: {sorted(cert_domains_set)}, " + f"desired: {sorted(desired_domains_set)}), asking new certificate..." ) continue - use_letsencrypt_staging = ( - getenv(f"{original_first_server}_USE_LETS_ENCRYPT_STAGING", "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_STAGING", "no") + # Check if CA provider has changed + ca_provider = (getenv(f"{original_first_server}_ACME_SSL_CA_PROVIDER", + "letsencrypt") if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")) + + renewal_file = DATA_PATH.joinpath("renewal", f"{first_server}.conf") + if renewal_file.is_file(): + current_server = None + with renewal_file.open("r") as file: + for line in file: + if line.startswith("server"): + current_server = line.strip().split("=", 1)[1].strip() + break + + expected_config = get_certificate_authority_config( + ca_provider, + (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_STAGING", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes" + ) + + if current_server and current_server != expected_config["server"]: + domains_to_ask[first_server] = 2 + LOGGER.warning(f"[{original_first_server}] CA provider for " + f"{first_server} has changed, asking new certificate...") + continue + + use_staging = ( + getenv(f"{original_first_server}_USE_LETS_ENCRYPT_STAGING", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_STAGING", "no") ) == "yes" is_test_cert = "TEST_CERT" in cert_domains.group("expiry_date") - if (is_test_cert and not use_letsencrypt_staging) or (not is_test_cert and use_letsencrypt_staging): + if (is_test_cert and not use_staging) or (not is_test_cert and use_staging): domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] Certificate environment (staging/production) changed for {first_server}, asking new certificate...") + LOGGER.warning(f"[{original_first_server}] Certificate environment " + f"(staging/production) changed for {first_server}, " + "asking new certificate...") continue - letsencrypt_provider = getenv(f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") if IS_MULTISITE else getenv("LETS_ENCRYPT_DNS_PROVIDER", "") + provider = (getenv(f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROVIDER", "")) - renewal_file = DATA_PATH.joinpath("renewal", f"{first_server}.conf") if not renewal_file.is_file(): - LOGGER.error(f"[{original_first_server}] Renewal file for {first_server} not found, asking new certificate...") + LOGGER.error(f"[{original_first_server}] Renewal file for " + f"{first_server} not found, asking new certificate...") domains_to_ask[first_server] = 1 continue @@ -504,17 +1107,20 @@ def certbot_new( break if letsencrypt_challenge == "dns": - if letsencrypt_provider and current_provider != letsencrypt_provider: + if provider and current_provider != provider: domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] Provider for {first_server} is not the same as in the certificate, asking new certificate...") + LOGGER.warning(f"[{original_first_server}] Provider for " + f"{first_server} is not the same as in the " + "certificate, asking new certificate...") continue # Check if DNS credentials have changed - if letsencrypt_provider and current_provider == letsencrypt_provider: - credential_key = f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + if provider and current_provider == provider: + credential_key = (f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM") current_credential_items = {} - # Collect current credential items for env_key, env_value in environ.items(): if env_value and env_key.startswith(credential_key): if " " not in env_value: @@ -522,106 +1128,162 @@ def certbot_new( continue key, value = env_value.split(" ", 1) current_credential_items[key.lower()] = ( - value.removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + value.removeprefix("= ").replace("\\n", "\n") + .replace("\\t", "\t").replace("\\r", "\r").strip() ) if "json_data" in current_credential_items: value = current_credential_items.pop("json_data") - if not current_credential_items and len(value) % 4 == 0 and match(r"^[A-Za-z0-9+/=]+$", value): + if (not current_credential_items and len(value) % 4 == 0 + and match(r"^[A-Za-z0-9+/=]+$", value)): with suppress(BaseException): decoded = b64decode(value).decode("utf-8") json_data = loads(decoded) if isinstance(json_data, dict): current_credential_items = { - k.lower(): str(v).removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + k.lower(): str(v).removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() for k, v in json_data.items() } if current_credential_items: - # Process regular credentials for base64 decoding for key, value in current_credential_items.items(): - if letsencrypt_provider != "rfc2136" and len(value) % 4 == 0 and match(r"^[A-Za-z0-9+/=]+$", value): + if (provider != "rfc2136" and len(value) % 4 == 0 + and match(r"^[A-Za-z0-9+/=]+$", value)): with suppress(BaseException): decoded = b64decode(value).decode("utf-8") if decoded != value: current_credential_items[key] = ( - decoded.removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + decoded.removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() ) - # Generate current credentials content - if letsencrypt_provider in provider_classes: + if provider in provider_classes: with suppress(ValidationError, KeyError): - current_provider_instance = provider_classes[letsencrypt_provider](**current_credential_items) - current_credentials_content = current_provider_instance.get_formatted_credentials() + current_provider_instance = provider_classes[provider]( + **current_credential_items + ) + current_credentials_content = ( + current_provider_instance.get_formatted_credentials() + ) - # Check if stored credentials file exists and compare file_type = current_provider_instance.get_file_type() - stored_credentials_path = CACHE_PATH.joinpath(first_server, f"credentials.{file_type}") + stored_credentials_path = CACHE_PATH.joinpath( + first_server, f"credentials.{file_type}" + ) if stored_credentials_path.is_file(): - stored_credentials_content = stored_credentials_path.read_bytes() + stored_credentials_content = ( + stored_credentials_path.read_bytes() + ) if stored_credentials_content != current_credentials_content: domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] DNS credentials for {first_server} have changed, asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] DNS credentials " + f"for {first_server} have changed, " + "asking new certificate..." + ) continue elif current_provider != "manual" and letsencrypt_challenge == "http": domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] {first_server} is no longer using DNS challenge, asking new certificate...") + LOGGER.warning(f"[{original_first_server}] {first_server} is no longer " + "using DNS challenge, asking new certificate...") continue domains_to_ask[first_server] = 0 - LOGGER.info(f"[{original_first_server}] Certificates already exist for domain(s) {domains}, expiry date: {cert_domains.group('expiry_date')}") + LOGGER.info(f"[{original_first_server}] Certificates already exist for " + f"domain(s) {domains}, expiry date: " + f"{cert_domains.group('expiry_date')}") psl_lines = None psl_rules = None for first_server, domains in domains_server_names.items(): - if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": - LOGGER.info(f"Let's Encrypt is not activated for {first_server}, skipping...") + if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": + LOGGER.info(f"SSL certificate generation is not activated for " + f"{first_server}, skipping...") continue - # * Getting all the necessary data + # Getting all the necessary data data = { - "email": (getenv(f"{first_server}_EMAIL_LETS_ENCRYPT", "") if IS_MULTISITE else getenv("EMAIL_LETS_ENCRYPT", "")) or f"contact@{first_server}", - "challenge": getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") if IS_MULTISITE else getenv("LETS_ENCRYPT_CHALLENGE", "http"), - "staging": (getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes", - "use_wildcard": (getenv(f"{first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes", - "provider": getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") if IS_MULTISITE else getenv("LETS_ENCRYPT_DNS_PROVIDER", ""), - "propagation": ( - getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION", "default") if IS_MULTISITE else getenv("LETS_ENCRYPT_DNS_PROPAGATION", "default") - ), - "profile": getenv(f"{first_server}_LETS_ENCRYPT_PROFILE", "classic") if IS_MULTISITE else getenv("LETS_ENCRYPT_PROFILE", "classic"), - "check_psl": ( - getenv(f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") if IS_MULTISITE else getenv("LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") - ) - == "no", - "max_retries": getenv(f"{first_server}_LETS_ENCRYPT_MAX_RETRIES", "0") if IS_MULTISITE else getenv("LETS_ENCRYPT_MAX_RETRIES", "0"), + "email": (getenv(f"{first_server}_EMAIL_LETS_ENCRYPT", "") if IS_MULTISITE + else getenv("EMAIL_LETS_ENCRYPT", "")) or f"contact@{first_server}", + "challenge": (getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_CHALLENGE", "http")), + "staging": (getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes", + "use_wildcard": (getenv(f"{first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes", + "provider": (getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROVIDER", "")), + "propagation": (getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION", + "default") if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROPAGATION", "default")), + "profile": (getenv(f"{first_server}_LETS_ENCRYPT_PROFILE", "classic") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_PROFILE", "classic")), + "check_psl": (getenv(f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", + "yes") if IS_MULTISITE + else getenv("LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes")) == "no", + "max_retries": (getenv(f"{first_server}_LETS_ENCRYPT_MAX_RETRIES", "0") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_MAX_RETRIES", "0")), + "ca_provider": (getenv(f"{first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")), + "api_key": (getenv(f"{first_server}_ACME_ZEROSSL_API_KEY", "") if IS_MULTISITE + else getenv("ACME_ZEROSSL_API_KEY", "")), "credential_items": {}, } + + LOGGER.info(f"Service {first_server} configuration:") + LOGGER.info(f" CA Provider: {data['ca_provider']}") + LOGGER.info(f" API Key provided: {'Yes' if data['api_key'] else 'No'}") + LOGGER.info(f" Challenge type: {data['challenge']}") + LOGGER.info(f" Staging: {data['staging']}") + LOGGER.info(f" Wildcard: {data['use_wildcard']}") # Override profile if custom profile is set - custom_profile = (getenv(f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE", "") if IS_MULTISITE else getenv("LETS_ENCRYPT_CUSTOM_PROFILE", "")).strip() + custom_profile = (getenv(f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_CUSTOM_PROFILE", "")).strip() if custom_profile: data["profile"] = custom_profile if data["challenge"] == "http" and data["use_wildcard"]: - LOGGER.warning(f"Wildcard is not supported with HTTP challenge, disabling wildcard for service {first_server}...") + LOGGER.warning(f"Wildcard is not supported with HTTP challenge, " + f"disabling wildcard for service {first_server}...") data["use_wildcard"] = False if (not data["use_wildcard"] and not domains_to_ask.get(first_server)) or ( - data["use_wildcard"] and not domains_to_ask.get(WILDCARD_GENERATOR.extract_wildcards_from_domains((first_server,))[0].lstrip("*.")) + data["use_wildcard"] and not domains_to_ask.get( + WILDCARD_GENERATOR.extract_wildcards_from_domains((first_server,))[0] + .lstrip("*.") + ) ): continue if not data["max_retries"].isdigit(): - LOGGER.warning(f"Invalid max retries value for service {first_server} : {data['max_retries']}, using default value of 0...") + LOGGER.warning(f"Invalid max retries value for service {first_server} : " + f"{data['max_retries']}, using default value of 0...") data["max_retries"] = 0 else: data["max_retries"] = int(data["max_retries"]) - # * Getting the DNS provider data if necessary + # Getting the DNS provider data if necessary if data["challenge"] == "dns": - credential_key = f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + credential_key = (f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM") credential_items = {} # Collect all credential items @@ -631,65 +1293,95 @@ def certbot_new( credential_items["json_data"] = env_value continue key, value = env_value.split(" ", 1) - credential_items[key.lower()] = value.removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + credential_items[key.lower()] = (value.removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip()) if "json_data" in credential_items: value = credential_items.pop("json_data") # Handle the case of a single credential that might be base64-encoded JSON - if not credential_items and len(value) % 4 == 0 and match(r"^[A-Za-z0-9+/=]+$", value): + if (not credential_items and len(value) % 4 == 0 + and match(r"^[A-Za-z0-9+/=]+$", value)): try: decoded = b64decode(value).decode("utf-8") json_data = loads(decoded) if isinstance(json_data, dict): data["credential_items"] = { - k.lower(): str(v).removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + k.lower(): str(v).removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() for k, v in json_data.items() } except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while decoding JSON data for service {first_server} : {value} : \n{e}") + LOGGER.error(f"Error while decoding JSON data for service " + f"{first_server} : {value} : \n{e}") if not data["credential_items"]: # Process regular credentials data["credential_items"] = {} for key, value in credential_items.items(): # Check for base64 encoding - if data["provider"] != "rfc2136" and len(value) % 4 == 0 and match(r"^[A-Za-z0-9+/=]+$", value): + if (data["provider"] != "rfc2136" and len(value) % 4 == 0 + and match(r"^[A-Za-z0-9+/=]+$", value)): try: decoded = b64decode(value).decode("utf-8") if decoded != value: - value = decoded.removeprefix("= ").replace("\\n", "\n").replace("\\t", "\t").replace("\\r", "\r").strip() + value = (decoded.removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip()) except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.debug(f"Error while decoding credential item {key} for service {first_server} : {value} : \n{e}") + LOGGER.debug(f"Error while decoding credential item {key} " + f"for service {first_server} : {value} : \n{e}") data["credential_items"][key] = value LOGGER.debug(f"Data for service {first_server} : {dumps(data)}") - # * Checking if the DNS data is valid + # Validate CA provider and API key requirements + LOGGER.info(f"Service {first_server} - CA Provider: {data['ca_provider']}, " + f"API Key provided: {'Yes' if data['api_key'] else 'No'}") + + if data["ca_provider"].lower() == "zerossl": + if not data["api_key"]: + LOGGER.warning(f"ZeroSSL API key not provided for service {first_server}, " + "falling back to Let's Encrypt...") + data["ca_provider"] = "letsencrypt" + else: + LOGGER.info(f"✓ ZeroSSL configuration valid for service {first_server}") + + # Checking if the DNS data is valid if data["challenge"] == "dns": if not data["provider"]: LOGGER.warning( - f"No provider found for service {first_server} (available providers : {', '.join(provider_classes.keys())}), skipping certificate(s) generation..." # noqa: E501 + f"No provider found for service {first_server} " + f"(available providers : {', '.join(provider_classes.keys())}), " + "skipping certificate(s) generation..." ) continue elif data["provider"] not in provider_classes: LOGGER.warning( - f"Provider {data['provider']} not found for service {first_server} (available providers : {', '.join(provider_classes.keys())}), skipping certificate(s) generation..." # noqa: E501 + f"Provider {data['provider']} not found for service {first_server} " + f"(available providers : {', '.join(provider_classes.keys())}), " + "skipping certificate(s) generation..." ) continue elif not data["credential_items"]: LOGGER.warning( - f"No valid credentials items found for service {first_server} (you should have at least one), skipping certificate(s) generation..." + f"No valid credentials items found for service {first_server} " + "(you should have at least one), skipping certificate(s) generation..." ) continue - # * Validating the credentials try: provider = provider_classes[data["provider"]](**data["credential_items"]) except ValidationError as ve: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while validating credentials for service {first_server} :\n{ve}") + LOGGER.error(f"Error while validating credentials for service " + f"{first_server} :\n{ve}") continue content = provider.get_formatted_credentials() @@ -698,9 +1390,10 @@ def certbot_new( is_blacklisted = False - # * Adding the domains to Wildcard Generator if necessary - file_type = provider.get_file_type() if data["challenge"] == "dns" else "txt" + # Adding the domains to Wildcard Generator if necessary + file_type = (provider.get_file_type() if data["challenge"] == "dns" else "txt") file_path = (first_server, f"credentials.{file_type}") + if data["use_wildcard"]: # Use the improved method for generating consistent group names group = WILDCARD_GENERATOR.create_group_name( @@ -714,7 +1407,8 @@ def certbot_new( LOGGER.info( f"Service {first_server} is using wildcard, " - + ("the propagation time will be the provider's default and " if data["challenge"] == "dns" else "") + + ("the propagation time will be the provider's default and " + if data["challenge"] == "dns" else "") + "the email will be the same as the first domain that created the group..." ) @@ -724,20 +1418,25 @@ def certbot_new( if psl_rules is None: psl_rules = parse_psl(psl_lines) - wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains(domains.split(" ")) + wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( + domains.split(" ") + ) - LOGGER.debug(f"Wildcard domains for {first_server} : {wildcards}") + LOGGER.info(f"Wildcard domains for {first_server} : {wildcards}") for d in wildcards: if is_domain_blacklisted(d, psl_rules): - LOGGER.error(f"Wildcard domain {d} is blacklisted by Public Suffix List, refusing certificate request for {first_server}.") + LOGGER.error(f"Wildcard domain {d} is blacklisted by Public " + f"Suffix List, refusing certificate request for " + f"{first_server}.") is_blacklisted = True break if not is_blacklisted: - WILDCARD_GENERATOR.extend(group, domains.split(" "), data["email"], data["staging"]) + WILDCARD_GENERATOR.extend(group, domains.split(" "), data["email"], + data["staging"]) file_path = (f"{group}.{file_type}",) - LOGGER.debug(f"[{first_server}] Wildcard group {group}") + LOGGER.info(f"[{first_server}] Wildcard group {group}") elif data["check_psl"]: if psl_lines is None: psl_lines = load_public_suffix_list(JOB) @@ -746,48 +1445,61 @@ def certbot_new( for d in domains.split(): if is_domain_blacklisted(d, psl_rules): - LOGGER.error(f"Domain {d} is blacklisted by Public Suffix List, refusing certificate request for {first_server}.") + LOGGER.error(f"Domain {d} is blacklisted by Public Suffix List, " + f"refusing certificate request for {first_server}.") is_blacklisted = True break if is_blacklisted: continue - # * Generating the credentials file + # Generating the credentials file credentials_path = CACHE_PATH.joinpath(*file_path) if data["challenge"] == "dns": if not credentials_path.is_file(): cached, err = JOB.cache_file( - credentials_path.name, content, job_name="certbot-renew", service_id=first_server if not data["use_wildcard"] else "" + credentials_path.name, content, job_name="certbot-renew", + service_id=first_server if not data["use_wildcard"] else "" ) if not cached: - LOGGER.error(f"Error while saving service {first_server}'s credentials file in cache : {err}") + LOGGER.error(f"Error while saving service {first_server}'s " + f"credentials file in cache : {err}") continue - LOGGER.info(f"Successfully saved service {first_server}'s credentials file in cache") + LOGGER.info(f"Successfully saved service {first_server}'s " + "credentials file in cache") elif data["use_wildcard"]: - LOGGER.info(f"Service {first_server}'s wildcard credentials file has already been generated") + LOGGER.info(f"Service {first_server}'s wildcard credentials file " + "has already been generated") else: old_content = credentials_path.read_bytes() if old_content != content: - LOGGER.warning(f"Service {first_server}'s credentials file is outdated, updating it...") - cached, err = JOB.cache_file(credentials_path.name, content, job_name="certbot-renew", service_id=first_server) + LOGGER.warning(f"Service {first_server}'s credentials file is " + "outdated, updating it...") + cached, err = JOB.cache_file(credentials_path.name, content, + job_name="certbot-renew", + service_id=first_server) if not cached: - LOGGER.error(f"Error while updating service {first_server}'s credentials file in cache : {err}") + LOGGER.error(f"Error while updating service {first_server}'s " + f"credentials file in cache : {err}") continue - LOGGER.info(f"Successfully updated service {first_server}'s credentials file in cache") + LOGGER.info(f"Successfully updated service {first_server}'s " + "credentials file in cache") else: LOGGER.info(f"Service {first_server}'s credentials file is up to date") credential_paths.add(credentials_path) - credentials_path.chmod(0o600) # ? Setting the permissions to 600 (this is important to avoid warnings from certbot) + credentials_path.chmod(0o600) # Setting the permissions to 600 (this is important to avoid warnings from certbot) if data["use_wildcard"]: continue domains = domains.replace(" ", ",") + ca_name = get_certificate_authority_config(data["ca_provider"])["name"] LOGGER.info( - f"Asking certificates for domain(s) : {domains} (email = {data['email']}){' using staging' if data['staging'] else ''} with {data['challenge']} challenge, using {data['profile']!r} profile..." + f"Asking {ca_name} certificates for domain(s) : {domains} " + f"(email = {data['email']}){' using staging' if data['staging'] else ''} " + f"with {data['challenge']} challenge, using {data['profile']!r} profile..." ) if ( @@ -803,6 +1515,9 @@ def certbot_new( domains_to_ask[first_server] == 2, cmd_env=env, max_retries=data["max_retries"], + ca_provider=data["ca_provider"], + api_key=data["api_key"], + server_name=first_server, ) != 0 ): @@ -814,20 +1529,40 @@ def certbot_new( generated_domains.update(domains.split(",")) - # * Generating the wildcards if necessary + # Generating the wildcards if necessary wildcards = WILDCARD_GENERATOR.get_wildcards() if wildcards: for group, data in wildcards.items(): if not data: continue - # * Generating the certificate from the generated credentials + + # Generating the certificate from the generated credentials group_parts = group.split("_") provider = group_parts[0] profile = group_parts[2] base_domain = group_parts[3] email = data.pop("email") - credentials_file = CACHE_PATH.joinpath(f"{group}.{provider_classes[provider].get_file_type() if provider in provider_classes else 'txt'}") + credentials_file = CACHE_PATH.joinpath( + f"{group}.{provider_classes[provider].get_file_type() if provider in provider_classes else 'txt'}" + ) + + # Get CA provider for this group + original_server = None + for server in domains_server_names.keys(): + if base_domain in server or server in base_domain: + original_server = server + break + + ca_provider = "letsencrypt" # default + api_key = None + if original_server: + ca_provider = (getenv(f"{original_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")) + api_key = (getenv(f"{original_server}_ACME_ZEROSSL_API_KEY", "") + if IS_MULTISITE + else getenv("ACME_ZEROSSL_API_KEY", "")) # Process different environment types (staging/prod) for key, domains in data.items(): @@ -835,9 +1570,11 @@ def certbot_new( continue staging = key == "staging" + ca_name = get_certificate_authority_config(ca_provider)["name"] LOGGER.info( - f"Asking wildcard certificates for domain(s): {domains} (email = {email})" - f"{' using staging ' if staging else ''} with {'dns' if provider in provider_classes else 'http'} challenge, " + f"Asking {ca_name} wildcard certificates for domain(s): {domains} " + f"(email = {email}){' using staging ' if staging else ''} " + f"with {'dns' if provider in provider_classes else 'http'} challenge, " f"using {profile!r} profile..." ) @@ -861,6 +1598,9 @@ def certbot_new( staging, domains_to_ask.get(base_domain, 0) == 2, cmd_env=env, + ca_provider=ca_provider, + api_key=api_key, + server_name=original_server, ) != 0 ): @@ -875,19 +1615,21 @@ def certbot_new( LOGGER.info("No wildcard domains found, skipping wildcard certificate(s) generation...") if CACHE_PATH.is_dir(): - # * Clearing all missing credentials files + # Clearing all missing credentials files for ext in ("*.ini", "*.env", "*.json"): for file in list(CACHE_PATH.rglob(ext)): if "etc" in file.parts or not file.is_file(): continue - # ? If the file is not in the wildcard groups, remove it + # If the file is not in the wildcard groups, remove it if file not in credential_paths: - LOGGER.debug(f"Removing old credentials file {file}") - JOB.del_cache(file.name, job_name="certbot-renew", service_id=file.parent.name if file.parent.name != "letsencrypt" else "") + LOGGER.info(f"Removing old credentials file {file}") + JOB.del_cache(file.name, job_name="certbot-renew", + service_id=file.parent.name if file.parent.name != "letsencrypt" else "") - # * Clearing all no longer needed certificates + # Clearing all no longer needed certificates if getenv("LETS_ENCRYPT_CLEAR_OLD_CERTS", "no") == "yes": - LOGGER.info("Clear old certificates is activated, removing old / no longer used certificates...") + LOGGER.info("Clear old certificates is activated, removing old / no longer " + "used certificates...") # Get list of all certificates proc = run( @@ -916,10 +1658,11 @@ def certbot_new( # Skip certificates that are in our active list if cert_name in active_cert_names: - LOGGER.debug(f"Keeping active certificate: {cert_name}") + LOGGER.info(f"Keeping active certificate: {cert_name}") continue - LOGGER.warning(f"Removing old certificate {cert_name} (not in active certificates list)") + LOGGER.warning(f"Removing old certificate {cert_name} " + "(not in active certificates list)") # Use certbot's delete command delete_proc = run( @@ -934,7 +1677,7 @@ def certbot_new( LOGS_DIR, "--cert-name", cert_name, - "-n", # non-interactive + "-n", ], stdin=DEVNULL, stdout=PIPE, @@ -946,7 +1689,6 @@ def certbot_new( if delete_proc.returncode == 0: LOGGER.info(f"Successfully deleted certificate {cert_name}") - # Remove any remaining files for this certificate cert_dir = DATA_PATH.joinpath("live", cert_name) archive_dir = DATA_PATH.joinpath("archive", cert_name) renewal_file = DATA_PATH.joinpath("renewal", f"{cert_name}.conf") @@ -967,19 +1709,22 @@ def certbot_new( renewal_file.unlink() LOGGER.info(f"Removed renewal file {renewal_file}") except Exception as e: - LOGGER.error(f"Failed to remove renewal file {renewal_file}: {e}") + LOGGER.error(f"Failed to remove renewal file " + f"{renewal_file}: {e}") else: - LOGGER.error(f"Failed to delete certificate {cert_name}: {delete_proc.stdout}") + LOGGER.error(f"Failed to delete certificate {cert_name}: " + f"{delete_proc.stdout}") else: LOGGER.error(f"Error listing certificates: {proc.stdout}") - # * Save data to db cache + # Save data to db cache if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): cached, err = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") if not cached: LOGGER.error(f"Error while saving data to db cache : {err}") else: LOGGER.info("Successfully saved data to db cache") + except SystemExit as e: status = e.code except BaseException as e: @@ -987,4 +1732,4 @@ def certbot_new( LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-new.py :\n{e}") -sys_exit(status) +sys_exit(status) \ No newline at end of file From 1d325571839850c4cde0a7a5bf8fb0ef264407e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sat, 28 Jun 2025 19:10:14 +0200 Subject: [PATCH 02/15] update inline documentation + linebreaks 80/120 --- .../core/letsencrypt/jobs/certbot-new.py | 974 +++++++++++++----- 1 file changed, 706 insertions(+), 268 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index d28ebdbb98..3c1abb15ec 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -54,7 +54,8 @@ from logger import setup_logger # type: ignore LOGGER = setup_logger("LETS-ENCRYPT.new") -CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot") +CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", + "bin", "certbot") DEPS_PATH = join(sep, "usr", "share", "bunkerweb", "deps", "python") LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.new.certbot") @@ -72,41 +73,62 @@ # ZeroSSL Configuration ZEROSSL_ACME_SERVER = "https://acme.zerossl.com/v2/DV90" -ZEROSSL_STAGING_SERVER = "https://acme.zerossl.com/v2/DV90" # ZeroSSL doesn't have staging +ZEROSSL_STAGING_SERVER = "https://acme.zerossl.com/v2/DV90" LETSENCRYPT_ACME_SERVER = "https://acme-v02.api.letsencrypt.org/directory" -LETSENCRYPT_STAGING_SERVER = "https://acme-staging-v02.api.letsencrypt.org/directory" +LETSENCRYPT_STAGING_SERVER = ( + "https://acme-staging-v02.api.letsencrypt.org/directory" +) +# Load and cache the public suffix list for domain validation. +# Fetches the PSL from the official source and caches it locally. +# Returns cached version if available and fresh (less than 1 day old). +# Args: job - Job instance for caching operations +# Returns: list - Lines from the public suffix list file def load_public_suffix_list(job): - # Load and cache the public suffix list for domain validation + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Loading public suffix list from cache or {PSL_URL}") + job_cache = job.get_cache(PSL_STATIC_FILE, with_info=True, with_data=True) if ( isinstance(job_cache, dict) and job_cache.get("last_update") - and job_cache["last_update"] < (datetime.now().astimezone() - - timedelta(days=1)).timestamp() + and job_cache["last_update"] < ( + datetime.now().astimezone() - timedelta(days=1) + ).timestamp() ): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Using cached public suffix list") return job_cache["data"].decode("utf-8").splitlines() try: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Downloading fresh PSL from {PSL_URL}") resp = get(PSL_URL, timeout=5) resp.raise_for_status() content = resp.text cached, err = JOB.cache_file(PSL_STATIC_FILE, content.encode("utf-8")) if not cached: - LOGGER.error(f"Error while saving public suffix list to cache : {err}") + LOGGER.error(f"Error while saving public suffix list to cache: {err}") return content.splitlines() except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while downloading public suffix list : {e}") + LOGGER.error(f"Error while downloading public suffix list: {e}") if PSL_STATIC_FILE.exists(): with PSL_STATIC_FILE.open("r", encoding="utf-8") as f: return f.read().splitlines() return [] +# Parse PSL lines into rules and exceptions sets. +# Processes the public suffix list format, handling comments, +# exceptions (lines starting with !), and regular rules. +# Args: psl_lines - List of lines from the PSL file +# Returns: dict - Contains 'rules' and 'exceptions' sets def parse_psl(psl_lines): - # Parse PSL lines into rules and exceptions sets + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Parsing {len(psl_lines)} PSL lines") + rules = set() exceptions = set() for line in psl_lines: @@ -117,16 +139,31 @@ def parse_psl(psl_lines): exceptions.add(line[1:]) continue rules.add(line) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Parsed {len(rules)} rules and {len(exceptions)} exceptions") + return {"rules": rules, "exceptions": exceptions} +# Check if domain is forbidden by PSL rules. +# Validates whether a domain would be blacklisted according to the +# Public Suffix List rules and exceptions. +# Args: domain - Domain name to check +# psl - Parsed PSL data (dict with 'rules' and 'exceptions') +# Returns: bool - True if domain is blacklisted def is_domain_blacklisted(domain, psl): - # Check if domain is forbidden by PSL rules domain = domain.lower().strip(".") labels = domain.split(".") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking domain {domain} against PSL rules") + for i in range(len(labels)): candidate = ".".join(labels[i:]) if candidate in psl["exceptions"]: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} allowed by PSL exception {candidate}") return False if candidate in psl["rules"]: if i == 0: @@ -143,8 +180,16 @@ def is_domain_blacklisted(domain, psl): return False +# Get ACME server configuration for the specified CA provider. +# Returns the appropriate ACME server URL and name for the given +# certificate authority and environment (staging/production). +# Args: ca_provider - Certificate authority name ('zerossl' or 'letsencrypt') +# staging - Whether to use staging environment +# Returns: dict - Server URL and CA name def get_certificate_authority_config(ca_provider, staging=False): - # Get ACME server configuration for the specified CA provider + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Getting CA config for {ca_provider}, staging={staging}") + if ca_provider.lower() == "zerossl": return { "server": ZEROSSL_STAGING_SERVER if staging else ZEROSSL_ACME_SERVER, @@ -152,20 +197,31 @@ def get_certificate_authority_config(ca_provider, staging=False): } else: # Default to Let's Encrypt return { - "server": LETSENCRYPT_STAGING_SERVER if staging else LETSENCRYPT_ACME_SERVER, + "server": (LETSENCRYPT_STAGING_SERVER if staging + else LETSENCRYPT_ACME_SERVER), "name": "Let's Encrypt" } +# Setup External Account Binding (EAB) credentials for ZeroSSL. +# Contacts the ZeroSSL API to obtain EAB credentials required for +# ACME certificate issuance with ZeroSSL. +# Args: email - Email address for the account +# api_key - ZeroSSL API key +# Returns: tuple - (eab_kid, eab_hmac_key) or (None, None) on failure def setup_zerossl_eab_credentials(email, api_key=None): - # Setup External Account Binding (EAB) credentials for ZeroSSL LOGGER.info(f"Setting up ZeroSSL EAB credentials for email: {email}") if not api_key: LOGGER.error("❌ ZeroSSL API key not provided") - LOGGER.warning("ZeroSSL API key not provided, attempting registration with email") + LOGGER.warning( + "ZeroSSL API key not provided, attempting registration with email" + ) return None, None + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Making request to ZeroSSL API for EAB credentials") + LOGGER.info("Making request to ZeroSSL API for EAB credentials") # Try the correct ZeroSSL API endpoint @@ -176,11 +232,17 @@ def setup_zerossl_eab_credentials(email, api_key=None): headers={"Authorization": f"Bearer {api_key}"}, timeout=30 ) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ZeroSSL API response status: {response.status_code}") LOGGER.info(f"ZeroSSL API response status: {response.status_code}") if response.status_code == 200: response.raise_for_status() eab_data = response.json() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ZeroSSL API response data: {eab_data}") LOGGER.info(f"ZeroSSL API response data: {eab_data}") # ZeroSSL typically returns eab_kid and eab_hmac_key directly @@ -189,15 +251,23 @@ def setup_zerossl_eab_credentials(email, api_key=None): eab_hmac_key = eab_data.get("eab_hmac_key") LOGGER.info(f"✓ Successfully obtained EAB credentials from ZeroSSL") LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") - LOGGER.info(f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}...") + LOGGER.info( + f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}..." + ) return eab_kid, eab_hmac_key else: LOGGER.error(f"❌ Invalid ZeroSSL API response format: {eab_data}") return None, None else: # Try alternative endpoint if first one fails - LOGGER.warning(f"Primary endpoint failed with {response.status_code}, trying alternative") + LOGGER.warning( + f"Primary endpoint failed with {response.status_code}, " + "trying alternative" + ) response_text = response.text + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Primary endpoint response: {response_text}") LOGGER.info(f"Primary endpoint response: {response_text}") # Try alternative endpoint with email parameter @@ -207,18 +277,32 @@ def setup_zerossl_eab_credentials(email, api_key=None): headers={"Authorization": f"Bearer {api_key}"}, timeout=30 ) - LOGGER.info(f"Alternative ZeroSSL API response status: {response.status_code}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Alternative ZeroSSL API response status: {response.status_code}" + ) + LOGGER.info( + f"Alternative ZeroSSL API response status: {response.status_code}" + ) response.raise_for_status() eab_data = response.json() + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Alternative ZeroSSL API response data: {eab_data}") LOGGER.info(f"Alternative ZeroSSL API response data: {eab_data}") if eab_data.get("success"): eab_kid = eab_data.get("eab_kid") eab_hmac_key = eab_data.get("eab_hmac_key") - LOGGER.info(f"✓ Successfully obtained EAB credentials from ZeroSSL (alternative endpoint)") + LOGGER.info( + "✓ Successfully obtained EAB credentials from ZeroSSL " + "(alternative endpoint)" + ) LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") - LOGGER.info(f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}...") + LOGGER.info( + f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}..." + ) return eab_kid, eab_hmac_key else: LOGGER.error(f"❌ ZeroSSL EAB registration failed: {eab_data}") @@ -238,16 +322,23 @@ def setup_zerossl_eab_credentials(email, api_key=None): return None, None +# Get CAA records for a domain using dig command. +# Queries DNS CAA records to check certificate authority authorization. +# Returns None if dig command is not available. +# Args: domain - Domain name to query +# Returns: list or None - List of CAA record dicts or None if unavailable def get_caa_records(domain): - # Get CAA records for a domain using dig command - # Check if dig command is available if not which("dig"): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("dig command not available for CAA record checking") LOGGER.info("dig command not available for CAA record checking") return None try: # Use dig to query CAA records + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Querying CAA records for domain: {domain}") LOGGER.info(f"Querying CAA records for domain: {domain}") result = run( ["dig", "+short", domain, "CAA"], @@ -274,21 +365,48 @@ def get_caa_records(domain): 'tag': tag, 'value': value }) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Parsed {len(caa_records)} CAA records for domain {domain}") LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain {domain}") return caa_records else: - LOGGER.info(f"No CAA records found for domain {domain} (dig return code: {result.returncode})") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"No CAA records found for domain {domain} " + f"(dig return code: {result.returncode})" + ) + LOGGER.info( + f"No CAA records found for domain {domain} " + f"(dig return code: {result.returncode})" + ) return [] except BaseException as e: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Error querying CAA records for {domain}: {e}") LOGGER.info(f"Error querying CAA records for {domain}: {e}") return None +# Check if the CA provider is authorized by CAA records. +# Validates whether the certificate authority is permitted to issue +# certificates for the domain according to CAA DNS records. +# Args: domain - Domain name to check +# ca_provider - Certificate authority provider name +# is_wildcard - Whether this is for a wildcard certificate +# Returns: bool - True if CA is authorized or no CAA restrictions exist def check_caa_authorization(domain, ca_provider, is_wildcard=False): - # Check if the CA provider is authorized by CAA records + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Checking CAA authorization for domain: {domain}, " + f"CA: {ca_provider}, wildcard: {is_wildcard}" + ) - LOGGER.info(f"Checking CAA authorization for domain: {domain}, CA: {ca_provider}, wildcard: {is_wildcard}") + LOGGER.info( + f"Checking CAA authorization for domain: {domain}, " + f"CA: {ca_provider}, wildcard: {is_wildcard}" + ) # Map CA providers to their CAA identifiers ca_identifiers = { @@ -304,10 +422,16 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Check CAA records for the domain and parent domains check_domain = domain.lstrip("*.") domain_parts = check_domain.split(".") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Will check CAA records for domain chain: {check_domain}") LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") for i in range(len(domain_parts)): current_domain = ".".join(domain_parts[i:]) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking CAA records for: {current_domain}") LOGGER.info(f"Checking CAA records for: {current_domain}") caa_records = get_caa_records(current_domain) @@ -331,8 +455,12 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Log found records if issue_records: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CAA issue records: {', '.join(issue_records)}") LOGGER.info(f"CAA issue records: {', '.join(issue_records)}") if issuewild_records: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CAA issuewild records: {', '.join(issuewild_records)}") LOGGER.info(f"CAA issuewild records: {', '.join(issuewild_records)}") # Check authorization based on certificate type @@ -345,34 +473,64 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): check_records = issue_records record_type = "issue" + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using CAA {record_type} records for authorization check") LOGGER.info(f"Using CAA {record_type} records for authorization check") if not check_records: - LOGGER.info(f"No relevant CAA {record_type} records found for {current_domain}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"No relevant CAA {record_type} records found for {current_domain}" + ) + LOGGER.info( + f"No relevant CAA {record_type} records found for {current_domain}" + ) continue # Check if any of our CA identifiers are authorized authorized = False - LOGGER.info(f"Checking authorization for CA identifiers: {', '.join(allowed_identifiers)}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Checking authorization for CA identifiers: " + f"{', '.join(allowed_identifiers)}" + ) + LOGGER.info( + f"Checking authorization for CA identifiers: " + f"{', '.join(allowed_identifiers)}" + ) for identifier in allowed_identifiers: for record in check_records: # Handle explicit deny (empty value or semicolon) if record == ";" or record.strip() == "": - LOGGER.warning(f"CAA {record_type} record explicitly denies all CAs") + LOGGER.warning( + f"CAA {record_type} record explicitly denies all CAs" + ) return False # Check for CA authorization if identifier in record: authorized = True - LOGGER.info(f"✓ CA {ca_provider} ({identifier}) authorized by CAA {record_type} record") + LOGGER.info( + f"✓ CA {ca_provider} ({identifier}) authorized by " + f"CAA {record_type} record" + ) break if authorized: break if not authorized: - LOGGER.error(f"❌ CA {ca_provider} is NOT authorized by CAA {record_type} records") - LOGGER.error(f"Domain {current_domain} CAA {record_type} allows: {', '.join(check_records)}") - LOGGER.error(f"But {ca_provider} uses: {', '.join(allowed_identifiers)}") + LOGGER.error( + f"❌ CA {ca_provider} is NOT authorized by " + f"CAA {record_type} records" + ) + LOGGER.error( + f"Domain {current_domain} CAA {record_type} allows: " + f"{', '.join(check_records)}" + ) + LOGGER.error( + f"But {ca_provider} uses: {', '.join(allowed_identifiers)}" + ) return False # If we found CAA records and we're authorized, we can stop checking parent domains @@ -380,13 +538,31 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): return True # No CAA records found in the entire chain - LOGGER.info(f"No CAA records found for {check_domain} or parent domains - any CA allowed") + LOGGER.info( + f"No CAA records found for {check_domain} or parent domains - any CA allowed" + ) return True -def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", is_wildcard=False): - # Validate that all domains have valid A/AAAA records and CAA authorization for HTTP challenge - LOGGER.info(f"Validating {len(domains_list)} domains for HTTP challenge: {', '.join(domains_list)}") +# Validate that all domains have valid A/AAAA records and CAA authorization +# for HTTP challenge. +# Checks DNS resolution and certificate authority authorization for each +# domain in the list to ensure HTTP challenge will succeed. +# Args: domains_list - List of domain names to validate +# ca_provider - Certificate authority provider name +# is_wildcard - Whether this is for wildcard certificates +# Returns: bool - True if all domains are valid for HTTP challenge +def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", + is_wildcard=False): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Validating {len(domains_list)} domains for HTTP challenge: " + f"{', '.join(domains_list)}" + ) + LOGGER.info( + f"Validating {len(domains_list)} domains for HTTP challenge: " + f"{', '.join(domains_list)}" + ) invalid_domains = [] caa_blocked_domains = [] @@ -401,7 +577,9 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", if external_ips.get("ipv6"): LOGGER.info(f"Server external IPv6 address: {external_ips['ipv6']}") else: - LOGGER.warning("Could not determine server external IP - skipping IP match validation") + LOGGER.warning( + "Could not determine server external IP - skipping IP match validation" + ) for domain in domains_list: # Check DNS A/AAAA records with retry mechanism @@ -414,28 +592,47 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", if not check_caa_authorization(domain, ca_provider, is_wildcard): caa_blocked_domains.append(domain) else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") LOGGER.info(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") # Report results if invalid_domains: - LOGGER.error(f"The following domains do not have valid A/AAAA records and cannot be used " - f"for HTTP challenge: {', '.join(invalid_domains)}") - LOGGER.error("Please ensure domains resolve to this server before requesting certificates") + LOGGER.error( + f"The following domains do not have valid A/AAAA records and " + f"cannot be used for HTTP challenge: {', '.join(invalid_domains)}" + ) + LOGGER.error( + "Please ensure domains resolve to this server before requesting certificates" + ) return False if caa_blocked_domains: - LOGGER.error(f"The following domains have CAA records that block {ca_provider}: " - f"{', '.join(caa_blocked_domains)}") - LOGGER.error("Please update CAA records to authorize the certificate authority or use a different CA") + LOGGER.error( + f"The following domains have CAA records that block {ca_provider}: " + f"{', '.join(caa_blocked_domains)}" + ) + LOGGER.error( + "Please update CAA records to authorize the certificate authority " + "or use a different CA" + ) LOGGER.info("You can skip CAA checking by setting ACME_SKIP_CAA_CHECK=yes") return False - LOGGER.info(f"All domains have valid DNS records and CAA authorization for HTTP challenge: {', '.join(domains_list)}") + LOGGER.info( + f"All domains have valid DNS records and CAA authorization for " + f"HTTP challenge: {', '.join(domains_list)}" + ) return True +# Get the external/public IP addresses of this server (both IPv4 and IPv6). +# Queries multiple external services to determine the server's public +# IP addresses for DNS validation purposes. +# Returns: dict or None - Dict with 'ipv4' and 'ipv6' keys, or None if all fail def get_external_ip(): - # Get the external/public IP addresses of this server (both IPv4 and IPv6) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Getting external IP addresses for server") LOGGER.info("Getting external IP addresses for server") ipv4_services = [ "https://ipv4.icanhazip.com", @@ -453,6 +650,8 @@ def get_external_ip(): external_ips = {"ipv4": None, "ipv6": None} # Try to get IPv4 address + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Attempting to get external IPv4 address") LOGGER.info("Attempting to get external IPv4 address") for service in ipv4_services: try: @@ -474,15 +673,22 @@ def get_external_ip(): # Validate it's a proper IPv4 address getaddrinfo(ip, None, AF_INET) external_ips["ipv4"] = ip + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Successfully obtained external IPv4 address: {ip}") LOGGER.info(f"Successfully obtained external IPv4 address: {ip}") break except gaierror: continue except BaseException as e: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Failed to get IPv4 address from {service}: {e}") LOGGER.info(f"Failed to get IPv4 address from {service}: {e}") continue # Try to get IPv6 address + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Attempting to get external IPv6 address") LOGGER.info("Attempting to get external IPv6 address") for service in ipv6_services: try: @@ -502,24 +708,42 @@ def get_external_ip(): # Validate it's a proper IPv6 address getaddrinfo(ip, None, AF_INET6) external_ips["ipv6"] = ip + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Successfully obtained external IPv6 address: {ip}") LOGGER.info(f"Successfully obtained external IPv6 address: {ip}") break except gaierror: continue except BaseException as e: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Failed to get IPv6 address from {service}: {e}") LOGGER.info(f"Failed to get IPv6 address from {service}: {e}") continue if not external_ips["ipv4"] and not external_ips["ipv6"]: - LOGGER.warning("Could not determine external IP address (IPv4 or IPv6) from any service") + LOGGER.warning( + "Could not determine external IP address (IPv4 or IPv6) from any service" + ) return None - LOGGER.info(f"External IP detection completed - IPv4: {external_ips['ipv4'] or 'not found'}, IPv6: {external_ips['ipv6'] or 'not found'}") + LOGGER.info( + f"External IP detection completed - " + f"IPv4: {external_ips['ipv4'] or 'not found'}, " + f"IPv6: {external_ips['ipv6'] or 'not found'}" + ) return external_ips +# Check if domain has valid A/AAAA records for HTTP challenge. +# Validates DNS resolution and optionally checks if the domain's +# IP addresses match the server's external IPs. +# Args: domain - Domain name to check +# external_ips - Dict with server's external IPv4/IPv6 addresses +# Returns: bool - True if domain has valid DNS records def check_domain_a_record(domain, external_ips=None): - # Check if domain has valid A/AAAA records for HTTP challenge + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking DNS A/AAAA records for domain: {domain}") LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") try: # Remove wildcard prefix if present @@ -537,9 +761,25 @@ def check_domain_a_record(domain, external_ips=None): # Log found addresses if ipv4_addresses: - LOGGER.info(f"Domain {check_domain} IPv4 A records: {', '.join(ipv4_addresses[:3])}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Domain {check_domain} IPv4 A records: " + f"{', '.join(ipv4_addresses[:3])}" + ) + LOGGER.info( + f"Domain {check_domain} IPv4 A records: " + f"{', '.join(ipv4_addresses[:3])}" + ) if ipv6_addresses: - LOGGER.info(f"Domain {check_domain} IPv6 AAAA records: {', '.join(ipv6_addresses[:3])}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Domain {check_domain} IPv6 AAAA records: " + f"{', '.join(ipv6_addresses[:3])}" + ) + LOGGER.info( + f"Domain {check_domain} IPv6 AAAA records: " + f"{', '.join(ipv6_addresses[:3])}" + ) # Check if any record matches the external IPs if external_ips: @@ -549,21 +789,37 @@ def check_domain_a_record(domain, external_ips=None): # Check IPv4 match if external_ips.get("ipv4") and ipv4_addresses: if external_ips["ipv4"] in ipv4_addresses: - LOGGER.info(f"✓ Domain {check_domain} IPv4 A record matches server external IP ({external_ips['ipv4']})") + LOGGER.info( + f"✓ Domain {check_domain} IPv4 A record matches " + f"server external IP ({external_ips['ipv4']})" + ) ipv4_match = True else: - LOGGER.warning(f"⚠ Domain {check_domain} IPv4 A record does not match server external IP") - LOGGER.warning(f" Domain IPv4: {', '.join(ipv4_addresses)}") + LOGGER.warning( + f"⚠ Domain {check_domain} IPv4 A record does not " + "match server external IP" + ) + LOGGER.warning( + f" Domain IPv4: {', '.join(ipv4_addresses)}" + ) LOGGER.warning(f" Server IPv4: {external_ips['ipv4']}") # Check IPv6 match if external_ips.get("ipv6") and ipv6_addresses: if external_ips["ipv6"] in ipv6_addresses: - LOGGER.info(f"✓ Domain {check_domain} IPv6 AAAA record matches server external IP ({external_ips['ipv6']})") + LOGGER.info( + f"✓ Domain {check_domain} IPv6 AAAA record matches " + f"server external IP ({external_ips['ipv6']})" + ) ipv6_match = True else: - LOGGER.warning(f"⚠ Domain {check_domain} IPv6 AAAA record does not match server external IP") - LOGGER.warning(f" Domain IPv6: {', '.join(ipv6_addresses)}") + LOGGER.warning( + f"⚠ Domain {check_domain} IPv6 AAAA record does not " + "match server external IP" + ) + LOGGER.warning( + f" Domain IPv6: {', '.join(ipv6_addresses)}" + ) LOGGER.warning(f" Server IPv6: {external_ips['ipv6']}") # Determine if we have any matching records @@ -571,23 +827,37 @@ def check_domain_a_record(domain, external_ips=None): has_external_ip = external_ips.get("ipv4") or external_ips.get("ipv6") if has_external_ip and not has_any_match: - LOGGER.warning(f"⚠ Domain {check_domain} records do not match any server external IP") - LOGGER.warning(f" HTTP challenge may fail - ensure domain points to this server") + LOGGER.warning( + f"⚠ Domain {check_domain} records do not match " + "any server external IP" + ) + LOGGER.warning( + f" HTTP challenge may fail - ensure domain points to this server" + ) # Check if we should treat this as an error strict_ip_check = getenv("ACME_HTTP_STRICT_IP_CHECK", "no") == "yes" if strict_ip_check: - LOGGER.error(f"Strict IP check enabled - rejecting certificate request for {check_domain}") + LOGGER.error( + f"Strict IP check enabled - rejecting certificate " + f"request for {check_domain}" + ) return False LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") return True else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Domain {check_domain} validation failed - no DNS resolution" + ) LOGGER.info(f"Domain {check_domain} validation failed - no DNS resolution") LOGGER.warning(f"Domain {check_domain} does not resolve") return False except gaierror as e: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {check_domain} DNS resolution failed (gaierror): {e}") LOGGER.info(f"Domain {check_domain} DNS resolution failed (gaierror): {e}") LOGGER.warning(f"DNS resolution failed for domain {check_domain}: {e}") return False @@ -597,35 +867,24 @@ def check_domain_a_record(domain, external_ips=None): return False -def validate_domains_for_http_challenge(domains_list): - # Validate that all domains have valid A/AAAA records for HTTP challenge - LOGGER.info(f"Validating {len(domains_list)} domains for HTTP challenge: {', '.join(domains_list)}") - invalid_domains = [] - - # Get external IPs once for all domain checks - external_ips = get_external_ip() - if external_ips: - if external_ips.get("ipv4"): - LOGGER.info(f"Server external IPv4 address: {external_ips['ipv4']}") - if external_ips.get("ipv6"): - LOGGER.info(f"Server external IPv6 address: {external_ips['ipv6']}") - else: - LOGGER.warning("Could not determine server external IP - skipping IP match validation") - - for domain in domains_list: - if not check_domain_a_record(domain, external_ips): - invalid_domains.append(domain) - - if invalid_domains: - LOGGER.error(f"The following domains do not have valid A/AAAA records and cannot be used " - f"for HTTP challenge: {', '.join(invalid_domains)}") - LOGGER.error("Please ensure domains resolve to this server before requesting certificates") - return False - - LOGGER.info(f"All domains have valid DNS records for HTTP challenge: {', '.join(domains_list)}") - return True - - +# Execute certbot with retry mechanism. +# Wrapper around certbot_new that implements automatic retries with +# exponential backoff for failed certificate generation attempts. +# Args: challenge_type - Type of ACME challenge ('dns' or 'http') +# domains - Comma-separated list of domain names +# email - Email address for certificate registration +# provider - DNS provider name (for DNS challenge) +# credentials_path - Path to credentials file +# propagation - DNS propagation time in seconds +# profile - Certificate profile to use +# staging - Whether to use staging environment +# force - Force renewal of existing certificates +# cmd_env - Environment variables for certbot process +# max_retries - Maximum number of retry attempts +# ca_provider - Certificate authority provider +# api_key - API key for CA (if required) +# server_name - Server name for multisite configurations +# Returns: int - Exit code (0 for success) def certbot_new_with_retry( challenge_type: Literal["dns", "http"], domains: str, @@ -642,13 +901,17 @@ def certbot_new_with_retry( api_key: str = None, server_name: str = None, ) -> int: - # Execute certbot with retry mechanism attempt = 1 while attempt <= max_retries + 1: if attempt > 1: - LOGGER.warning(f"Certificate generation failed, retrying... " - f"(attempt {attempt}/{max_retries + 1})") + LOGGER.warning( + f"Certificate generation failed, retrying... " + f"(attempt {attempt}/{max_retries + 1})" + ) wait_time = min(30 * (2 ** (attempt - 2)), 300) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Waiting {wait_time} seconds before retry...") LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) @@ -682,6 +945,23 @@ def certbot_new_with_retry( return result +# Generate new certificate using certbot. +# Main function to request SSL/TLS certificates from a certificate authority +# using the ACME protocol via certbot. +# Args: challenge_type - Type of ACME challenge ('dns' or 'http') +# domains - Comma-separated list of domain names +# email - Email address for certificate registration +# provider - DNS provider name (for DNS challenge) +# credentials_path - Path to credentials file +# propagation - DNS propagation time in seconds +# profile - Certificate profile to use +# staging - Whether to use staging environment +# force - Force renewal of existing certificates +# cmd_env - Environment variables for certbot process +# ca_provider - Certificate authority provider +# api_key - API key for CA (if required) +# server_name - Server name for multisite configurations +# Returns: int - Exit code (0 for success) def certbot_new( challenge_type: Literal["dns", "http"], domains: str, @@ -697,12 +977,14 @@ def certbot_new( api_key: str = None, server_name: str = None, ) -> int: - # Generate new certificate using certbot if isinstance(credentials_path, str): credentials_path = Path(credentials_path) ca_config = get_certificate_authority_config(ca_provider, staging) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Building certbot command for {domains}") + command = [ CERTBOT_BIN, "certonly", @@ -731,25 +1013,38 @@ def certbot_new( if challenge_type == "dns" and provider in ("infomaniak", "ionos"): # Infomaniak and IONOS require RSA certificates with 4096-bit keys command.extend(["--rsa-key-size", "4096"]) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using RSA-4096 for {provider} provider with {domains}") LOGGER.info(f"Using RSA-4096 for {provider} provider with {domains}") else: # Use elliptic curve certificates for all other providers if ca_provider.lower() == "zerossl": # Use P-384 elliptic curve for ZeroSSL certificates command.extend(["--elliptic-curve", "secp384r1"]) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using ZeroSSL P-384 curve for {domains}") LOGGER.info(f"Using ZeroSSL P-384 curve for {domains}") else: # Use P-256 elliptic curve for Let's Encrypt certificates command.extend(["--elliptic-curve", "secp256r1"]) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using Let's Encrypt P-256 curve for {domains}") LOGGER.info(f"Using Let's Encrypt P-256 curve for {domains}") # Handle ZeroSSL EAB credentials if ca_provider.lower() == "zerossl": + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ZeroSSL detected as CA provider for {domains}") LOGGER.info(f"ZeroSSL detected as CA provider for {domains}") # Check for manually provided EAB credentials first - eab_kid_env = getenv("ACME_ZEROSSL_EAB_KID", "") or getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "") - eab_hmac_env = getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") + eab_kid_env = (getenv("ACME_ZEROSSL_EAB_KID", "") or + getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "")) + eab_hmac_env = (getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or + getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "")) if eab_kid_env and eab_hmac_env: LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials from environment") @@ -757,6 +1052,8 @@ def certbot_new( LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid_env[:10]}...") elif api_key: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ZeroSSL API key provided, setting up EAB credentials") LOGGER.info(f"ZeroSSL API key provided, setting up EAB credentials") eab_kid, eab_hmac = setup_zerossl_eab_credentials(email, api_key) if eab_kid and eab_hmac: @@ -765,12 +1062,17 @@ def certbot_new( LOGGER.info(f"EAB Kid: {eab_kid[:10]}...") else: LOGGER.error("❌ Failed to obtain ZeroSSL EAB credentials") - LOGGER.error("Alternative: Set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY environment variables") + LOGGER.error( + "Alternative: Set ACME_ZEROSSL_EAB_KID and " + "ACME_ZEROSSL_EAB_HMAC_KEY environment variables" + ) LOGGER.warning("Proceeding without EAB - this will likely fail") else: LOGGER.error("❌ No ZeroSSL API key provided!") LOGGER.error("Set ACME_ZEROSSL_API_KEY environment variable") - LOGGER.error("Or set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY directly") + LOGGER.error( + "Or set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY directly" + ) LOGGER.warning("Proceeding without EAB - this will likely fail") if challenge_type == "dns": @@ -778,8 +1080,10 @@ def certbot_new( if propagation != "default": if not propagation.isdigit(): - LOGGER.warning(f"Invalid propagation time : {propagation}, " - "using provider's default...") + LOGGER.warning( + f"Invalid propagation time: {propagation}, " + "using provider's default..." + ) else: command.extend([f"--dns-{provider}-propagation-seconds", propagation]) @@ -827,10 +1131,14 @@ def certbot_new( mask_next = True else: safe_command.append(item) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Command: {' '.join(safe_command)}") LOGGER.info(f"Command: {' '.join(safe_command)}") current_date = datetime.now() - process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, env=cmd_env) + process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, + env=cmd_env) while process.poll() is None: if process.stderr: @@ -841,10 +1149,14 @@ def certbot_new( break if datetime.now() - current_date > timedelta(seconds=5): - challenge_info = (" (this may take a while depending on the provider)" - if challenge_type == "dns" else "") - LOGGER.info(f"⏳ Still generating {ca_config['name']} certificate(s)" - f"{challenge_info}...") + challenge_info = ( + " (this may take a while depending on the provider)" + if challenge_type == "dns" else "" + ) + LOGGER.info( + f"⏳ Still generating {ca_config['name']} certificate(s)" + f"{challenge_info}..." + ) current_date = datetime.now() return process.returncode @@ -876,12 +1188,13 @@ def certbot_new( if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes": use_letsencrypt = True - if first_server and getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", - "http") == "dns": + if (first_server and + getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") == "dns"): use_letsencrypt_dns = True - domains_server_names[first_server] = getenv(f"{first_server}_SERVER_NAME", - first_server).lower() + domains_server_names[first_server] = getenv( + f"{first_server}_SERVER_NAME", first_server + ).lower() if not use_letsencrypt: LOGGER.info("Let's Encrypt is not activated, skipping generation...") @@ -942,12 +1255,13 @@ def certbot_new( "PATH": getenv("PATH", ""), "PYTHONPATH": getenv("PYTHONPATH", ""), "RELOAD_MIN_TIMEOUT": getenv("RELOAD_MIN_TIMEOUT", "5"), - "DISABLE_CONFIGURATION_TESTING": getenv("DISABLE_CONFIGURATION_TESTING", - "no").lower(), + "DISABLE_CONFIGURATION_TESTING": getenv( + "DISABLE_CONFIGURATION_TESTING", "no" + ).lower(), } - env["PYTHONPATH"] = env["PYTHONPATH"] + (f":{DEPS_PATH}" - if DEPS_PATH not in env["PYTHONPATH"] - else "") + env["PYTHONPATH"] = env["PYTHONPATH"] + ( + f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" + ) if getenv("DATABASE_URI"): env["DATABASE_URI"] = getenv("DATABASE_URI") @@ -978,17 +1292,18 @@ def certbot_new( active_cert_names = set() if proc.returncode != 0: - LOGGER.error(f"Error while checking certificates :\n{proc.stdout}") + LOGGER.error(f"Error while checking certificates:\n{proc.stdout}") else: certificate_blocks = stdout.split("Certificate Name: ")[1:] for first_server, domains in domains_server_names.items(): - if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": + if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): continue - letsencrypt_challenge = (getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", - "http") if IS_MULTISITE - else getenv("LETS_ENCRYPT_CHALLENGE", "http")) + letsencrypt_challenge = ( + getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") if IS_MULTISITE + else getenv("LETS_ENCRYPT_CHALLENGE", "http") + ) original_first_server = deepcopy(first_server) if ( @@ -997,13 +1312,16 @@ def certbot_new( if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes" ): - wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains((first_server,)) + wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( + (first_server,) + ) first_server = wildcards[0].lstrip("*.") domains = set(wildcards) else: domains = set(domains.split(" ")) - # Add the certificate name to our active set regardless if we're generating it or not + # Add the certificate name to our active set regardless + # if we're generating it or not active_cert_names.add(first_server) certificate_block = None @@ -1014,29 +1332,41 @@ def certbot_new( if not certificate_block: domains_to_ask[first_server] = 1 - LOGGER.warning(f"[{original_first_server}] Certificate block for " - f"{first_server} not found, asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] Certificate block for " + f"{first_server} not found, asking new certificate..." + ) continue # Validating the credentials try: - cert_domains = search(r"Domains: (?P.*)\n\s*Expiry Date: " - r"(?P.*)\n", certificate_block, MULTILINE) + cert_domains = search( + r"Domains: (?P.*)\n\s*Expiry Date: " + r"(?P.*)\n", + certificate_block, + MULTILINE + ) except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"[{original_first_server}] Error while parsing " - f"certificate block: {e}") + LOGGER.error( + f"[{original_first_server}] Error while parsing " + f"certificate block: {e}" + ) continue if not cert_domains: - LOGGER.error(f"[{original_first_server}] Failed to parse domains " - "and expiry date from certificate block.") + LOGGER.error( + f"[{original_first_server}] Failed to parse domains " + "and expiry date from certificate block." + ) continue cert_domains_list = cert_domains.group("domains").strip().split() cert_domains_set = set(cert_domains_list) - desired_domains_set = (set(domains) if isinstance(domains, (list, set)) - else set(domains.split())) + desired_domains_set = ( + set(domains) if isinstance(domains, (list, set)) + else set(domains.split()) + ) if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 @@ -1048,9 +1378,11 @@ def certbot_new( continue # Check if CA provider has changed - ca_provider = (getenv(f"{original_first_server}_ACME_SSL_CA_PROVIDER", - "letsencrypt") if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")) + ca_provider = ( + getenv(f"{original_first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") + ) renewal_file = DATA_PATH.joinpath("renewal", f"{first_server}.conf") if renewal_file.is_file(): @@ -1070,8 +1402,10 @@ def certbot_new( if current_server and current_server != expected_config["server"]: domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] CA provider for " - f"{first_server} has changed, asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] CA provider for " + f"{first_server} has changed, asking new certificate..." + ) continue use_staging = ( @@ -1083,18 +1417,24 @@ def certbot_new( if (is_test_cert and not use_staging) or (not is_test_cert and use_staging): domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] Certificate environment " - f"(staging/production) changed for {first_server}, " - "asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] Certificate environment " + f"(staging/production) changed for {first_server}, " + "asking new certificate..." + ) continue - provider = (getenv(f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROVIDER", "")) + provider = ( + getenv(f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROVIDER", "") + ) if not renewal_file.is_file(): - LOGGER.error(f"[{original_first_server}] Renewal file for " - f"{first_server} not found, asking new certificate...") + LOGGER.error( + f"[{original_first_server}] Renewal file for " + f"{first_server} not found, asking new certificate..." + ) domains_to_ask[first_server] = 1 continue @@ -1109,16 +1449,20 @@ def certbot_new( if letsencrypt_challenge == "dns": if provider and current_provider != provider: domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] Provider for " - f"{first_server} is not the same as in the " - "certificate, asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] Provider for " + f"{first_server} is not the same as in the " + "certificate, asking new certificate..." + ) continue # Check if DNS credentials have changed if provider and current_provider == provider: - credential_key = (f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" - if IS_MULTISITE - else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM") + credential_key = ( + f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + ) current_credential_items = {} for env_key, env_value in environ.items(): @@ -1180,7 +1524,8 @@ def certbot_new( stored_credentials_content = ( stored_credentials_path.read_bytes() ) - if stored_credentials_content != current_credentials_content: + if (stored_credentials_content != + current_credentials_content): domains_to_ask[first_server] = 2 LOGGER.warning( f"[{original_first_server}] DNS credentials " @@ -1190,61 +1535,92 @@ def certbot_new( continue elif current_provider != "manual" and letsencrypt_challenge == "http": domains_to_ask[first_server] = 2 - LOGGER.warning(f"[{original_first_server}] {first_server} is no longer " - "using DNS challenge, asking new certificate...") + LOGGER.warning( + f"[{original_first_server}] {first_server} is no longer " + "using DNS challenge, asking new certificate..." + ) continue domains_to_ask[first_server] = 0 - LOGGER.info(f"[{original_first_server}] Certificates already exist for " - f"domain(s) {domains}, expiry date: " - f"{cert_domains.group('expiry_date')}") + LOGGER.info( + f"[{original_first_server}] Certificates already exist for " + f"domain(s) {domains}, expiry date: " + f"{cert_domains.group('expiry_date')}" + ) psl_lines = None psl_rules = None for first_server, domains in domains_server_names.items(): - if (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes": - LOGGER.info(f"SSL certificate generation is not activated for " - f"{first_server}, skipping...") + if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): + LOGGER.info( + f"SSL certificate generation is not activated for " + f"{first_server}, skipping..." + ) continue # Getting all the necessary data data = { - "email": (getenv(f"{first_server}_EMAIL_LETS_ENCRYPT", "") if IS_MULTISITE - else getenv("EMAIL_LETS_ENCRYPT", "")) or f"contact@{first_server}", - "challenge": (getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_CHALLENGE", "http")), - "staging": (getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes", - "use_wildcard": (getenv(f"{first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes", - "provider": (getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROVIDER", "")), - "propagation": (getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION", - "default") if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROPAGATION", "default")), - "profile": (getenv(f"{first_server}_LETS_ENCRYPT_PROFILE", "classic") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_PROFILE", "classic")), - "check_psl": (getenv(f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", - "yes") if IS_MULTISITE - else getenv("LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes")) == "no", - "max_retries": (getenv(f"{first_server}_LETS_ENCRYPT_MAX_RETRIES", "0") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_MAX_RETRIES", "0")), - "ca_provider": (getenv(f"{first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") - if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")), - "api_key": (getenv(f"{first_server}_ACME_ZEROSSL_API_KEY", "") if IS_MULTISITE - else getenv("ACME_ZEROSSL_API_KEY", "")), + "email": ( + getenv(f"{first_server}_EMAIL_LETS_ENCRYPT", "") if IS_MULTISITE + else getenv("EMAIL_LETS_ENCRYPT", "") + ) or f"contact@{first_server}", + "challenge": ( + getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_CHALLENGE", "http") + ), + "staging": ( + getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_STAGING", "no") + ) == "yes", + "use_wildcard": ( + getenv(f"{first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_WILDCARD", "no") + ) == "yes", + "provider": ( + getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROVIDER", "") + ), + "propagation": ( + getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION", "default") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DNS_PROPAGATION", "default") + ), + "profile": ( + getenv(f"{first_server}_LETS_ENCRYPT_PROFILE", "classic") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_PROFILE", "classic") + ), + "check_psl": ( + getenv(f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") + ) == "no", + "max_retries": ( + getenv(f"{first_server}_LETS_ENCRYPT_MAX_RETRIES", "0") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_MAX_RETRIES", "0") + ), + "ca_provider": ( + getenv(f"{first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") + ), + "api_key": ( + getenv(f"{first_server}_ACME_ZEROSSL_API_KEY", "") if IS_MULTISITE + else getenv("ACME_ZEROSSL_API_KEY", "") + ), "credential_items": {}, } + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Service {first_server} configuration: {data}") + LOGGER.info(f"Service {first_server} configuration:") LOGGER.info(f" CA Provider: {data['ca_provider']}") LOGGER.info(f" API Key provided: {'Yes' if data['api_key'] else 'No'}") @@ -1253,15 +1629,19 @@ def certbot_new( LOGGER.info(f" Wildcard: {data['use_wildcard']}") # Override profile if custom profile is set - custom_profile = (getenv(f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE", "") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_CUSTOM_PROFILE", "")).strip() + custom_profile = ( + getenv(f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE", "") + if IS_MULTISITE + else getenv("LETS_ENCRYPT_CUSTOM_PROFILE", "") + ).strip() if custom_profile: data["profile"] = custom_profile if data["challenge"] == "http" and data["use_wildcard"]: - LOGGER.warning(f"Wildcard is not supported with HTTP challenge, " - f"disabling wildcard for service {first_server}...") + LOGGER.warning( + f"Wildcard is not supported with HTTP challenge, " + f"disabling wildcard for service {first_server}..." + ) data["use_wildcard"] = False if (not data["use_wildcard"] and not domains_to_ask.get(first_server)) or ( @@ -1273,17 +1653,21 @@ def certbot_new( continue if not data["max_retries"].isdigit(): - LOGGER.warning(f"Invalid max retries value for service {first_server} : " - f"{data['max_retries']}, using default value of 0...") + LOGGER.warning( + f"Invalid max retries value for service {first_server}: " + f"{data['max_retries']}, using default value of 0..." + ) data["max_retries"] = 0 else: data["max_retries"] = int(data["max_retries"]) # Getting the DNS provider data if necessary if data["challenge"] == "dns": - credential_key = (f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" - if IS_MULTISITE - else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM") + credential_key = ( + f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + ) credential_items = {} # Collect all credential items @@ -1293,10 +1677,12 @@ def certbot_new( credential_items["json_data"] = env_value continue key, value = env_value.split(" ", 1) - credential_items[key.lower()] = (value.removeprefix("= ") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\r", "\r").strip()) + credential_items[key.lower()] = ( + value.removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() + ) if "json_data" in credential_items: value = credential_items.pop("json_data") @@ -1316,8 +1702,10 @@ def certbot_new( } except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while decoding JSON data for service " - f"{first_server} : {value} : \n{e}") + LOGGER.error( + f"Error while decoding JSON data for service " + f"{first_server}: {value} : \n{e}" + ) if not data["credential_items"]: # Process regular credentials @@ -1329,26 +1717,35 @@ def certbot_new( try: decoded = b64decode(value).decode("utf-8") if decoded != value: - value = (decoded.removeprefix("= ") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\r", "\r").strip()) + value = ( + decoded.removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() + ) except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.debug(f"Error while decoding credential item {key} " - f"for service {first_server} : {value} : \n{e}") + LOGGER.debug( + f"Error while decoding credential item {key} " + f"for service {first_server}: {value} : \n{e}" + ) data["credential_items"][key] = value - LOGGER.debug(f"Data for service {first_server} : {dumps(data)}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Data for service {first_server}: {dumps(data)}") # Validate CA provider and API key requirements - LOGGER.info(f"Service {first_server} - CA Provider: {data['ca_provider']}, " - f"API Key provided: {'Yes' if data['api_key'] else 'No'}") + LOGGER.info( + f"Service {first_server} - CA Provider: {data['ca_provider']}, " + f"API Key provided: {'Yes' if data['api_key'] else 'No'}" + ) if data["ca_provider"].lower() == "zerossl": if not data["api_key"]: - LOGGER.warning(f"ZeroSSL API key not provided for service {first_server}, " - "falling back to Let's Encrypt...") + LOGGER.warning( + f"ZeroSSL API key not provided for service {first_server}, " + "falling back to Let's Encrypt..." + ) data["ca_provider"] = "letsencrypt" else: LOGGER.info(f"✓ ZeroSSL configuration valid for service {first_server}") @@ -1358,14 +1755,14 @@ def certbot_new( if not data["provider"]: LOGGER.warning( f"No provider found for service {first_server} " - f"(available providers : {', '.join(provider_classes.keys())}), " + f"(available providers: {', '.join(provider_classes.keys())}), " "skipping certificate(s) generation..." ) continue elif data["provider"] not in provider_classes: LOGGER.warning( f"Provider {data['provider']} not found for service {first_server} " - f"(available providers : {', '.join(provider_classes.keys())}), " + f"(available providers: {', '.join(provider_classes.keys())}), " "skipping certificate(s) generation..." ) continue @@ -1380,8 +1777,10 @@ def certbot_new( provider = provider_classes[data["provider"]](**data["credential_items"]) except ValidationError as ve: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while validating credentials for service " - f"{first_server} :\n{ve}") + LOGGER.error( + f"Error while validating credentials for service " + f"{first_server}:\n{ve}" + ) continue content = provider.get_formatted_credentials() @@ -1422,19 +1821,22 @@ def certbot_new( domains.split(" ") ) - LOGGER.info(f"Wildcard domains for {first_server} : {wildcards}") + LOGGER.info(f"Wildcard domains for {first_server}: {wildcards}") for d in wildcards: if is_domain_blacklisted(d, psl_rules): - LOGGER.error(f"Wildcard domain {d} is blacklisted by Public " - f"Suffix List, refusing certificate request for " - f"{first_server}.") + LOGGER.error( + f"Wildcard domain {d} is blacklisted by Public " + f"Suffix List, refusing certificate request for " + f"{first_server}." + ) is_blacklisted = True break if not is_blacklisted: - WILDCARD_GENERATOR.extend(group, domains.split(" "), data["email"], - data["staging"]) + WILDCARD_GENERATOR.extend( + group, domains.split(" "), data["email"], data["staging"] + ) file_path = (f"{group}.{file_type}",) LOGGER.info(f"[{first_server}] Wildcard group {group}") elif data["check_psl"]: @@ -1445,8 +1847,10 @@ def certbot_new( for d in domains.split(): if is_domain_blacklisted(d, psl_rules): - LOGGER.error(f"Domain {d} is blacklisted by Public Suffix List, " - f"refusing certificate request for {first_server}.") + LOGGER.error( + f"Domain {d} is blacklisted by Public Suffix List, " + f"refusing certificate request for {first_server}." + ) is_blacklisted = True break @@ -1463,33 +1867,50 @@ def certbot_new( service_id=first_server if not data["use_wildcard"] else "" ) if not cached: - LOGGER.error(f"Error while saving service {first_server}'s " - f"credentials file in cache : {err}") + LOGGER.error( + f"Error while saving service {first_server}'s " + f"credentials file in cache: {err}" + ) continue - LOGGER.info(f"Successfully saved service {first_server}'s " - "credentials file in cache") + LOGGER.info( + f"Successfully saved service {first_server}'s " + "credentials file in cache" + ) elif data["use_wildcard"]: - LOGGER.info(f"Service {first_server}'s wildcard credentials file " - "has already been generated") + LOGGER.info( + f"Service {first_server}'s wildcard credentials file " + "has already been generated" + ) else: old_content = credentials_path.read_bytes() if old_content != content: - LOGGER.warning(f"Service {first_server}'s credentials file is " - "outdated, updating it...") - cached, err = JOB.cache_file(credentials_path.name, content, - job_name="certbot-renew", - service_id=first_server) + LOGGER.warning( + f"Service {first_server}'s credentials file is " + "outdated, updating it..." + ) + cached, err = JOB.cache_file( + credentials_path.name, content, + job_name="certbot-renew", + service_id=first_server + ) if not cached: - LOGGER.error(f"Error while updating service {first_server}'s " - f"credentials file in cache : {err}") + LOGGER.error( + f"Error while updating service {first_server}'s " + f"credentials file in cache: {err}" + ) continue - LOGGER.info(f"Successfully updated service {first_server}'s " - "credentials file in cache") + LOGGER.info( + f"Successfully updated service {first_server}'s " + "credentials file in cache" + ) else: - LOGGER.info(f"Service {first_server}'s credentials file is up to date") + LOGGER.info( + f"Service {first_server}'s credentials file is up to date" + ) credential_paths.add(credentials_path) - credentials_path.chmod(0o600) # Setting the permissions to 600 (this is important to avoid warnings from certbot) + # Setting the permissions to 600 (this is important to avoid warnings from certbot) + credentials_path.chmod(0o600) if data["use_wildcard"]: continue @@ -1497,7 +1918,7 @@ def certbot_new( domains = domains.replace(" ", ",") ca_name = get_certificate_authority_config(data["ca_provider"])["name"] LOGGER.info( - f"Asking {ca_name} certificates for domain(s) : {domains} " + f"Asking {ca_name} certificates for domain(s): {domains} " f"(email = {data['email']}){' using staging' if data['staging'] else ''} " f"with {data['challenge']} challenge, using {data['profile']!r} profile..." ) @@ -1522,10 +1943,10 @@ def certbot_new( != 0 ): status = 2 - LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...") + LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") else: status = 1 if status == 0 else status - LOGGER.info(f"Certificate generation succeeded for domain(s) : {domains}") + LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") generated_domains.update(domains.split(",")) @@ -1557,12 +1978,16 @@ def certbot_new( ca_provider = "letsencrypt" # default api_key = None if original_server: - ca_provider = (getenv(f"{original_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") - if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt")) - api_key = (getenv(f"{original_server}_ACME_ZEROSSL_API_KEY", "") - if IS_MULTISITE - else getenv("ACME_ZEROSSL_API_KEY", "")) + ca_provider = ( + getenv(f"{original_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + if IS_MULTISITE + else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") + ) + api_key = ( + getenv(f"{original_server}_ACME_ZEROSSL_API_KEY", "") + if IS_MULTISITE + else getenv("ACME_ZEROSSL_API_KEY", "") + ) # Process different environment types (staging/prod) for key, domains in data.items(): @@ -1605,14 +2030,16 @@ def certbot_new( != 0 ): status = 2 - LOGGER.error(f"Certificate generation failed for domain(s) {domains} ...") + LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") else: status = 1 if status == 0 else status LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") generated_domains.update(domains_split) else: - LOGGER.info("No wildcard domains found, skipping wildcard certificate(s) generation...") + LOGGER.info( + "No wildcard domains found, skipping wildcard certificate(s) generation..." + ) if CACHE_PATH.is_dir(): # Clearing all missing credentials files @@ -1623,13 +2050,18 @@ def certbot_new( # If the file is not in the wildcard groups, remove it if file not in credential_paths: LOGGER.info(f"Removing old credentials file {file}") - JOB.del_cache(file.name, job_name="certbot-renew", - service_id=file.parent.name if file.parent.name != "letsencrypt" else "") + JOB.del_cache( + file.name, job_name="certbot-renew", + service_id=(file.parent.name + if file.parent.name != "letsencrypt" else "") + ) # Clearing all no longer needed certificates if getenv("LETS_ENCRYPT_CLEAR_OLD_CERTS", "no") == "yes": - LOGGER.info("Clear old certificates is activated, removing old / no longer " - "used certificates...") + LOGGER.info( + "Clear old certificates is activated, removing old / no longer " + "used certificates..." + ) # Get list of all certificates proc = run( @@ -1661,8 +2093,10 @@ def certbot_new( LOGGER.info(f"Keeping active certificate: {cert_name}") continue - LOGGER.warning(f"Removing old certificate {cert_name} " - "(not in active certificates list)") + LOGGER.warning( + f"Removing old certificate {cert_name} " + "(not in active certificates list)" + ) # Use certbot's delete command delete_proc = run( @@ -1709,11 +2143,15 @@ def certbot_new( renewal_file.unlink() LOGGER.info(f"Removed renewal file {renewal_file}") except Exception as e: - LOGGER.error(f"Failed to remove renewal file " - f"{renewal_file}: {e}") + LOGGER.error( + f"Failed to remove renewal file " + f"{renewal_file}: {e}" + ) else: - LOGGER.error(f"Failed to delete certificate {cert_name}: " - f"{delete_proc.stdout}") + LOGGER.error( + f"Failed to delete certificate {cert_name}: " + f"{delete_proc.stdout}" + ) else: LOGGER.error(f"Error listing certificates: {proc.stdout}") @@ -1721,7 +2159,7 @@ def certbot_new( if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): cached, err = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") if not cached: - LOGGER.error(f"Error while saving data to db cache : {err}") + LOGGER.error(f"Error while saving data to db cache: {err}") else: LOGGER.info("Successfully saved data to db cache") @@ -1730,6 +2168,6 @@ def certbot_new( except BaseException as e: status = 1 LOGGER.debug(format_exc()) - LOGGER.error(f"Exception while running certbot-new.py :\n{e}") + LOGGER.error(f"Exception while running certbot-new.py:\n{e}") sys_exit(status) \ No newline at end of file From d54ae1e7d935ac74325327c431db677292a641fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sat, 28 Jun 2025 19:27:12 +0200 Subject: [PATCH 03/15] update inline documentation + linebreaks 80/120 --- .../core/letsencrypt/jobs/certbot-auth.py | 133 ++++- .../core/letsencrypt/jobs/certbot-cleanup.py | 129 ++++- .../core/letsencrypt/jobs/certbot-deploy.py | 180 +++++- .../core/letsencrypt/jobs/certbot-renew.py | 196 ++++++- .../core/letsencrypt/jobs/letsencrypt.py | 539 +++++++++++++----- 5 files changed, 959 insertions(+), 218 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-auth.py b/src/common/core/letsencrypt/jobs/certbot-auth.py index d1483b3208..559d2da2d4 100644 --- a/src/common/core/letsencrypt/jobs/certbot-auth.py +++ b/src/common/core/letsencrypt/jobs/certbot-auth.py @@ -6,7 +6,8 @@ from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) + for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -19,40 +20,144 @@ status = 0 try: - # Get env vars + # Get environment variables for ACME HTTP challenge + # CERTBOT_TOKEN: The filename for the challenge file + # CERTBOT_VALIDATION: The content to write to the challenge file token = getenv("CERTBOT_TOKEN", "") validation = getenv("CERTBOT_VALIDATION", "") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ACME HTTP challenge authentication started") + LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") + LOGGER.debug(f"Validation length: {len(validation) if validation else 0} chars") + + # Detect the current BunkerWeb integration type + # This determines how we handle the challenge deployment integration = get_integration() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Integration detection completed: {integration}") LOGGER.info(f"Detected {integration} integration") - # Cluster case + # Cluster case: Docker, Swarm, Kubernetes, Autoconf + # For cluster deployments, we need to distribute the challenge + # to all instances via the BunkerWeb API if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Cluster integration detected, initializing database connection") + + # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) + # Get all active BunkerWeb instances from the database instances = db.get_instances() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Retrieved {len(instances)} instances from database") + for i, instance in enumerate(instances): + LOGGER.debug( + f"Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" + ) LOGGER.info(f"Sending challenge to {len(instances)} instances") + + # Send challenge to each instance via API for instance in instances: - api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"]) - sent, err, status, resp = api.request("POST", "/lets-encrypt/challenge", data={"token": token, "validation": validation}) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Sending challenge to instance: " + f"{instance['hostname']}:{instance['port']}" + ) + + # Create API client for this instance + api = API( + f"http://{instance['hostname']}:{instance['port']}", + host=instance["server_name"] + ) + + # Send POST request to deploy the challenge + sent, err, status_code, resp = api.request( + "POST", + "/lets-encrypt/challenge", + data={"token": token, "validation": validation} + ) + if not sent: status = 1 - LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") - elif status != 200: + LOGGER.error( + f"Can't send API request to " + f"{api.endpoint}/lets-encrypt/challenge: {err}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API request failed with error: {err}") + elif status_code != 200: status = 1 - LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error( + f"Error while sending API request to " + f"{api.endpoint}/lets-encrypt/challenge: " + f"status = {resp['status']}, msg = {resp['msg']}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API response details: {resp}") else: - LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") + LOGGER.info( + f"Successfully sent API request to " + f"{api.endpoint}/lets-encrypt/challenge" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Challenge successfully deployed to " + f"{instance['hostname']}:{instance['port']}" + ) - # Linux case + # Linux case: Standalone installation + # For standalone Linux installations, we write the challenge + # file directly to the local filesystem else: - root_dir = Path(sep, "var", "tmp", "bunkerweb", "lets-encrypt", ".well-known", "acme-challenge") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + "Standalone Linux integration detected, " + "writing challenge file locally" + ) + + # Create the ACME challenge directory structure + # This follows the standard .well-known/acme-challenge path + root_dir = Path( + sep, "var", "tmp", "bunkerweb", "lets-encrypt", + ".well-known", "acme-challenge" + ) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Creating challenge directory: {root_dir}") + + # Create directory structure with appropriate permissions root_dir.mkdir(parents=True, exist_ok=True) - root_dir.joinpath(token).write_text(validation, encoding="utf-8") + + # Write the challenge validation content to the token file + challenge_file = root_dir.joinpath(token) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Writing challenge file: {challenge_file}") + LOGGER.debug(f"Challenge file content length: {len(validation)} bytes") + + challenge_file.write_text(validation, encoding="utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Challenge file written successfully") + + LOGGER.info(f"Challenge file created at {challenge_file}") + except BaseException as e: status = 1 LOGGER.debug(format_exc()) - LOGGER.error(f"Exception while running certbot-auth.py :\n{e}") + LOGGER.error(f"Exception while running certbot-auth.py:\n{e}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Full exception traceback logged above") + +if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ACME HTTP challenge authentication completed with status: {status}") -sys_exit(status) +sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-cleanup.py b/src/common/core/letsencrypt/jobs/certbot-cleanup.py index 3e896b14a4..6509fdd212 100644 --- a/src/common/core/letsencrypt/jobs/certbot-cleanup.py +++ b/src/common/core/letsencrypt/jobs/certbot-cleanup.py @@ -6,7 +6,8 @@ from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) + for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -19,35 +20,137 @@ status = 0 try: - # Get env vars + # Get environment variables for ACME HTTP challenge cleanup + # CERTBOT_TOKEN: The filename of the challenge file to remove token = getenv("CERTBOT_TOKEN", "") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ACME HTTP challenge cleanup started") + LOGGER.debug(f"Token to clean: {token[:10] if token else 'None'}...") + + # Detect the current BunkerWeb integration type + # This determines how we handle the challenge cleanup process integration = get_integration() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Integration detection completed: {integration}") LOGGER.info(f"Detected {integration} integration") - # Cluster case + # Cluster case: Docker, Swarm, Kubernetes, Autoconf + # For cluster deployments, we need to remove the challenge + # from all instances via the BunkerWeb API if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + "Cluster integration detected, initializing database connection" + ) + + # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) + + # Get all active BunkerWeb instances from the database instances = db.get_instances() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Retrieved {len(instances)} instances from database") + for i, instance in enumerate(instances): + LOGGER.debug( + f"Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" + ) LOGGER.info(f"Cleaning challenge from {len(instances)} instances") + + # Remove challenge from each instance via API for instance in instances: - api = API(f"http://{instance['hostname']}:{instance['port']}", host=instance["server_name"]) - sent, err, status, resp = api.request("DELETE", "/lets-encrypt/challenge", data={"token": token}) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Cleaning challenge from instance: " + f"{instance['hostname']}:{instance['port']}" + ) + + # Create API client for this instance + api = API( + f"http://{instance['hostname']}:{instance['port']}", + host=instance["server_name"] + ) + + # Send DELETE request to remove the challenge + sent, err, status_code, resp = api.request( + "DELETE", + "/lets-encrypt/challenge", + data={"token": token} + ) + if not sent: status = 1 - LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/challenge : {err}") - elif status != 200: + LOGGER.error( + f"Can't send API request to " + f"{api.endpoint}/lets-encrypt/challenge: {err}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API request failed with error: {err}") + elif status_code != 200: status = 1 - LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/challenge : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error( + f"Error while sending API request to " + f"{api.endpoint}/lets-encrypt/challenge: " + f"status = {resp['status']}, msg = {resp['msg']}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API response details: {resp}") else: - LOGGER.info(f"Successfully sent API request to {api.endpoint}/lets-encrypt/challenge") - # Linux case + LOGGER.info( + f"Successfully sent API request to " + f"{api.endpoint}/lets-encrypt/challenge" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Challenge successfully cleaned from " + f"{instance['hostname']}:{instance['port']}" + ) + + # Linux case: Standalone installation + # For standalone Linux installations, we remove the challenge + # file directly from the local filesystem else: - Path(sep, "var", "tmp", "bunkerweb", "lets-encrypt", ".well-known", "acme-challenge", token).unlink(missing_ok=True) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + "Standalone Linux integration detected, " + "removing challenge file locally" + ) + + # Construct path to the ACME challenge file + # This follows the standard .well-known/acme-challenge path + challenge_file = Path( + sep, "var", "tmp", "bunkerweb", "lets-encrypt", + ".well-known", "acme-challenge", token + ) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Challenge file to remove: {challenge_file}") + LOGGER.debug(f"File exists before cleanup: {challenge_file.exists()}") + + # Remove the challenge file if it exists + # missing_ok=True prevents errors if file doesn't exist + challenge_file.unlink(missing_ok=True) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"File exists after cleanup: {challenge_file.exists()}") + LOGGER.debug("Local challenge file cleanup completed") + + LOGGER.info(f"Challenge file removed: {challenge_file}") + except BaseException as e: status = 1 LOGGER.debug(format_exc()) - LOGGER.error(f"Exception while running certbot-cleanup.py :\n{e}") + LOGGER.error(f"Exception while running certbot-cleanup.py:\n{e}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Full exception traceback logged above") + +if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"ACME HTTP challenge cleanup completed with status: {status}") -sys_exit(status) +sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index 2112511685..ba56d3ffad 100644 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -7,7 +7,8 @@ from tarfile import open as tar_open from traceback import format_exc -for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) + for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -19,24 +20,72 @@ status = 0 try: - # Get env vars + # Get environment variables for certificate deployment + # CERTBOT_TOKEN: Token from certbot (currently unused but preserved) + # RENEWED_DOMAINS: Domains that were successfully renewed token = getenv("CERTBOT_TOKEN", "") - - LOGGER.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful") - - # Create tarball of /var/cache/bunkerweb/letsencrypt + renewed_domains = getenv("RENEWED_DOMAINS", "") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate deployment started") + LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") + LOGGER.debug(f"Renewed domains: {renewed_domains}") + + LOGGER.info(f"Certificates renewal for {renewed_domains} successful") + + # Create tarball of certificate directory for distribution + # This packages all certificate files into a compressed archive + # for efficient transfer to cluster instances + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Creating certificate archive for distribution") + tgz = BytesIO() + cert_source_path = join(sep, "var", "cache", "bunkerweb", "letsencrypt", "etc") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Source certificate path: {cert_source_path}") + # Create compressed tarball containing certificate files + # compresslevel=3 provides good compression with reasonable performance with tar_open(mode="w:gz", fileobj=tgz, compresslevel=3) as tf: - tf.add(join(sep, "var", "cache", "bunkerweb", "letsencrypt", "etc"), arcname="etc") + tf.add(cert_source_path, arcname="etc") + + # Reset buffer position for reading tgz.seek(0, 0) files = {"archive.tar.gz": tgz} - + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate archive created, size: {tgz.getbuffer().nbytes} bytes") + + # Initialize database connection to get cluster instances + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Initializing database connection") + db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) + # Get all active BunkerWeb instances for certificate distribution instances = db.get_instances() - services = db.get_non_default_settings(global_only=True, methods=False, with_drafts=True, filtered_settings=("SERVER_NAME",))["SERVER_NAME"].split(" ") + + # Get server names to calculate appropriate reload timeout + # More services require longer timeout for configuration reload + services = db.get_non_default_settings( + global_only=True, + methods=False, + with_drafts=True, + filtered_settings=("SERVER_NAME",) + )["SERVER_NAME"].split(" ") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Retrieved {len(instances)} instances from database") + LOGGER.debug(f"Found {len(services)} configured services: {services}") + for i, instance in enumerate(instances): + LOGGER.debug( + f"Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" + ) + # Configure reload timeout based on environment and service count + # Minimum timeout prevents premature timeouts on slow systems reload_min_timeout = getenv("RELOAD_MIN_TIMEOUT", "5") if not reload_min_timeout.isdigit(): @@ -44,39 +93,122 @@ reload_min_timeout = 5 reload_min_timeout = int(reload_min_timeout) - - for instance in instances: + + # Calculate actual timeout: minimum timeout or 3 seconds per service + calculated_timeout = max(reload_min_timeout, 3 * len(services)) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Reload timeout configuration:") + LOGGER.debug(f" Minimum timeout: {reload_min_timeout}s") + LOGGER.debug(f" Service count: {len(services)}") + LOGGER.debug(f" Calculated timeout: {calculated_timeout}s") + + LOGGER.info(f"Deploying certificates to {len(instances)} instances") + + # Deploy certificates to each cluster instance + for i, instance in enumerate(instances): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing instance {i+1}/{len(instances)}") + + # Construct API endpoint for this instance endpoint = f"http://{instance['hostname']}:{instance['port']}" host = instance["server_name"] + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Connecting to: {endpoint} (host: {host})") + api = API(endpoint, host=host) - sent, err, status, resp = api.request("POST", "/lets-encrypt/certificates", files=files) + # Upload certificate archive to the instance + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Uploading certificate archive to {endpoint}") + + sent, err, status_code, resp = api.request( + "POST", + "/lets-encrypt/certificates", + files=files + ) + if not sent: status = 1 - LOGGER.error(f"Can't send API request to {api.endpoint}/lets-encrypt/certificates : {err}") - elif status != 200: + LOGGER.error( + f"Can't send API request to " + f"{api.endpoint}/lets-encrypt/certificates: {err}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate upload failed with error: {err}") + continue + elif status_code != 200: status = 1 - LOGGER.error(f"Error while sending API request to {api.endpoint}/lets-encrypt/certificates : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error( + f"Error while sending API request to " + f"{api.endpoint}/lets-encrypt/certificates: " + f"status = {resp['status']}, msg = {resp['msg']}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate upload response: {resp}") + continue else: LOGGER.info( - f"Successfully sent API request to {api.endpoint}/lets-encrypt/certificates", + f"Successfully sent API request to " + f"{api.endpoint}/lets-encrypt/certificates" ) - sent, err, status, resp = api.request( + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate archive uploaded successfully") + + # Trigger configuration reload on the instance + # Configuration testing can be disabled via environment variable + disable_testing = getenv("DISABLE_CONFIGURATION_TESTING", "no").lower() + test_config = "no" if disable_testing == "yes" else "yes" + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Triggering reload on {endpoint}") + LOGGER.debug(f"Configuration testing: {test_config}") + LOGGER.debug(f"Reload timeout: {calculated_timeout}s") + + sent, err, status_code, resp = api.request( "POST", - f"/reload?test={'no' if getenv('DISABLE_CONFIGURATION_TESTING', 'no').lower() == 'yes' else 'yes'}", - timeout=max(reload_min_timeout, 3 * len(services)), + f"/reload?test={test_config}", + timeout=calculated_timeout, ) + if not sent: status = 1 - LOGGER.error(f"Can't send API request to {api.endpoint}/reload : {err}") - elif status != 200: + LOGGER.error( + f"Can't send API request to {api.endpoint}/reload: {err}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Reload request failed with error: {err}") + elif status_code != 200: status = 1 - LOGGER.error(f"Error while sending API request to {api.endpoint}/reload : status = {resp['status']}, msg = {resp['msg']}") + LOGGER.error( + f"Error while sending API request to {api.endpoint}/reload: " + f"status = {resp['status']}, msg = {resp['msg']}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Reload response: {resp}") else: LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Instance {endpoint} reloaded successfully") + + # Reset file pointer for next instance + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Resetting archive buffer for next instance") + tgz.seek(0, 0) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Certificate deployment process completed") + except BaseException as e: status = 1 LOGGER.debug(format_exc()) - LOGGER.error(f"Exception while running certbot-deploy.py :\n{e}") + LOGGER.error(f"Exception while running certbot-deploy.py:\n{e}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Full exception traceback logged above") + +if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate deployment completed with status: {status}") -sys_exit(status) +sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 79cc7f7378..766b04abd0 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -7,7 +7,8 @@ from sys import exit as sys_exit, path as sys_path from traceback import format_exc -for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: +for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) + for paths in (("deps", "python"), ("utils",), ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -16,7 +17,8 @@ LOGGER = setup_logger("LETS-ENCRYPT.renew") LIB_PATH = Path(sep, "var", "lib", "bunkerweb", "letsencrypt") -CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot") +CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", + "bin", "certbot") DEPS_PATH = join(sep, "usr", "share", "bunkerweb", "deps", "python") LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.renew.certbot") @@ -28,72 +30,212 @@ LOGS_DIR = join(sep, "var", "log", "bunkerweb", "letsencrypt") try: - # Check if we're using let's encrypt + # Determine if Let's Encrypt is enabled in the current configuration + # This checks both single-site and multi-site deployment modes + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Starting Let's Encrypt certificate renewal process") + LOGGER.debug("Checking if Let's Encrypt is enabled") + use_letsencrypt = False + multisite_mode = getenv("MULTISITE", "no") == "yes" + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Multisite mode: {multisite_mode}") - if getenv("MULTISITE", "no") == "no": + # Single-site mode: Check global AUTO_LETS_ENCRYPT setting + if not multisite_mode: use_letsencrypt = getenv("AUTO_LETS_ENCRYPT", "no") == "yes" + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Single-site mode - AUTO_LETS_ENCRYPT: {use_letsencrypt}") + + # Multi-site mode: Check per-server AUTO_LETS_ENCRYPT settings else: - for first_server in getenv("SERVER_NAME", "www.example.com").split(" "): - if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes": - use_letsencrypt = True - break + server_names = getenv("SERVER_NAME", "www.example.com").split(" ") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Multi-site mode - checking {len(server_names)} servers") + LOGGER.debug(f"Server names: {server_names}") + + # Check if any server has Let's Encrypt enabled + for i, first_server in enumerate(server_names): + if first_server: + server_le_enabled = getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes" + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug( + f"Server {i+1} ({first_server}): " + f"AUTO_LETS_ENCRYPT = {server_le_enabled}" + ) + + if server_le_enabled: + use_letsencrypt = True + break + # Exit early if Let's Encrypt is not configured if not use_letsencrypt: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Let's Encrypt not enabled, exiting renewal process") LOGGER.info("Let's Encrypt is not activated, skipping renew...") sys_exit(0) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Let's Encrypt is enabled, proceeding with renewal") + + # Initialize job handler for caching operations + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Initializing job handler") + JOB = Job(LOGGER, __file__) + # Set up environment variables for certbot execution + # These control paths, timeouts, and configuration testing behavior + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Setting up environment for certbot execution") + env = { "PATH": getenv("PATH", ""), "PYTHONPATH": getenv("PYTHONPATH", ""), "RELOAD_MIN_TIMEOUT": getenv("RELOAD_MIN_TIMEOUT", "5"), - "DISABLE_CONFIGURATION_TESTING": getenv("DISABLE_CONFIGURATION_TESTING", "no").lower(), + "DISABLE_CONFIGURATION_TESTING": getenv( + "DISABLE_CONFIGURATION_TESTING", "no" + ).lower(), } - env["PYTHONPATH"] = env["PYTHONPATH"] + (f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "") + + # Ensure our Python dependencies are in the path + env["PYTHONPATH"] = env["PYTHONPATH"] + ( + f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" + ) + + # Pass database URI if configured (for cluster deployments) if getenv("DATABASE_URI"): env["DATABASE_URI"] = getenv("DATABASE_URI") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Environment configuration:") + LOGGER.debug(f" PATH: {env['PATH'][:100]}..." if len(env['PATH']) > 100 else f" PATH: {env['PATH']}") + LOGGER.debug(f" PYTHONPATH: {env['PYTHONPATH'][:100]}..." if len(env['PYTHONPATH']) > 100 else f" PYTHONPATH: {env['PYTHONPATH']}") + LOGGER.debug(f" RELOAD_MIN_TIMEOUT: {env['RELOAD_MIN_TIMEOUT']}") + LOGGER.debug(f" DISABLE_CONFIGURATION_TESTING: {env['DISABLE_CONFIGURATION_TESTING']}") + LOGGER.debug(f" DATABASE_URI configured: {'Yes' if getenv('DATABASE_URI') else 'No'}") + + # Construct certbot renew command with appropriate options + # --no-random-sleep-on-renew: Prevents random delays in scheduled runs + # Paths are configured to use BunkerWeb's certificate storage locations + command = [ + CERTBOT_BIN, + "renew", + "--no-random-sleep-on-renew", + "--config-dir", + DATA_PATH.as_posix(), + "--work-dir", + WORK_DIR, + "--logs-dir", + LOGS_DIR, + ] + + # Add verbose flag if debug logging is enabled + if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": + command.append("-v") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Debug mode enabled, adding verbose flag to certbot") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certbot command: {' '.join(command)}") + LOGGER.debug(f"Working directory: {WORK_DIR}") + LOGGER.debug(f"Config directory: {DATA_PATH.as_posix()}") + LOGGER.debug(f"Logs directory: {LOGS_DIR}") + + LOGGER.info("Starting certificate renewal process") + # Execute certbot renew command + # Process output is captured and logged through our logger + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Executing certbot renew command") + process = Popen( - [ - CERTBOT_BIN, - "renew", - "--no-random-sleep-on-renew", - "--config-dir", - DATA_PATH.as_posix(), - "--work-dir", - WORK_DIR, - "--logs-dir", - LOGS_DIR, - ] - + (["-v"] if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG" else []), + command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, env=env, ) + + # Stream certbot output to our logger in real-time + # This ensures all certbot messages are captured in BunkerWeb logs + line_count = 0 while process.poll() is None: if process.stderr: for line in process.stderr: + line_count += 1 LOGGER_CERTBOT.info(line.strip()) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG" and line_count % 10 == 0: + LOGGER.debug(f"Processed {line_count} certbot output lines") - if process.returncode != 0: + # Wait for process completion and check return code + final_return_code = process.returncode + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certbot process completed with return code: {final_return_code}") + LOGGER.debug(f"Total certbot output lines processed: {line_count}") + + # Handle renewal results + if final_return_code != 0: status = 2 LOGGER.error("Certificates renewal failed") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Renewal process failed, certificate data will not be cached") + else: + LOGGER.info("Certificate renewal completed successfully") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Renewal process succeeded, proceeding to cache certificate data") - # Save Let's Encrypt data to db cache + # Save Let's Encrypt certificate data to database cache + # This ensures certificate data is available for distribution to cluster nodes + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Checking if certificate data directory exists") + LOGGER.debug(f"Data path: {DATA_PATH}") + LOGGER.debug(f"Directory exists: {DATA_PATH.is_dir()}") + if DATA_PATH.is_dir(): + dir_contents = list(DATA_PATH.iterdir()) + LOGGER.debug(f"Directory contents count: {len(dir_contents)}") + + # Only cache if directory exists and contains files if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Caching certificate data to database") + cached, err = JOB.cache_dir(DATA_PATH) if not cached: - LOGGER.error(f"Error while saving Let's Encrypt data to db cache : {err}") + LOGGER.error( + f"Error while saving Let's Encrypt data to db cache: {err}" + ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Cache operation failed with error: {err}") else: LOGGER.info("Successfully saved Let's Encrypt data to db cache") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Certificate data successfully cached to database") + else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("No certificate data to cache (directory empty or missing)") + LOGGER.warning("No certificate data found to cache") + except SystemExit as e: status = e.code + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Script exiting with SystemExit code: {e.code}") except BaseException as e: status = 2 LOGGER.debug(format_exc()) - LOGGER.error(f"Exception while running certbot-renew.py :\n{e}") + LOGGER.error(f"Exception while running certbot-renew.py:\n{e}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Full exception traceback logged above") + LOGGER.debug("Setting exit status to 2 due to unexpected exception") + +if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate renewal process completed with status: {status}") -sys_exit(status) +sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/letsencrypt.py b/src/common/core/letsencrypt/jobs/letsencrypt.py index 18ed75da69..31a09a6e68 100644 --- a/src/common/core/letsencrypt/jobs/letsencrypt.py +++ b/src/common/core/letsencrypt/jobs/letsencrypt.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from os import getenv from pathlib import Path from sys import path as sys_path from typing import Dict, List, Literal, Optional @@ -15,37 +16,71 @@ sys_path.append(python_path_str) +# Factory function for creating a model_validator for alias mapping. +# This allows DNS providers to accept credentials under multiple field names +# for better compatibility with different configuration formats. +# Args: field_map - Dictionary mapping canonical field names to list of aliases +# Returns: Configured model_validator function def alias_model_validator(field_map: dict): - """Factory function for creating a `model_validator` for alias mapping.""" - def validator(cls, values): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Processing aliases for {cls.__name__}") + print(f"DEBUG: Input values: {list(values.keys())}") + for field, aliases in field_map.items(): for alias in aliases: if alias in values: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Mapping alias '{alias}' to field '{field}'") values[field] = values[alias] break + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Final mapped values: {list(values.keys())}") + return values return model_validator(mode="before")(validator) class Provider(BaseModel): - """Base class for DNS providers.""" + # Base class for DNS providers. + # Provides common functionality for credential formatting and file type handling. + # All DNS provider classes inherit from this base class. model_config = ConfigDict(extra="ignore") + # Return the formatted credentials to be written to a file. + # Default implementation creates INI-style key=value format. + # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: - """Return the formatted credentials to be written to a file.""" - return "\n".join(f"{key} = {value}" for key, value in self.model_dump(exclude={"file_type"}).items()).encode("utf-8") - + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + excluded_fields = {"file_type"} + fields = self.model_dump(exclude=excluded_fields) + print(f"DEBUG: {self.__class__.__name__} formatting {len(fields)} fields") + print(f"DEBUG: Excluded fields: {excluded_fields}") + + content = "\n".join( + f"{key} = {value}" + for key, value in self.model_dump(exclude={"file_type"}).items() + ).encode("utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(content)} bytes of credential content") + + return content + + # Return the file type that the credentials should be written to. + # Default implementation returns 'ini' for most providers. + # Returns: Literal type indicating file extension @staticmethod def get_file_type() -> Literal["ini"]: - """Return the file type that the credentials should be written to.""" return "ini" class CloudflareProvider(Provider): - """Cloudflare DNS provider.""" + # Supports both API token (recommended) and legacy email/API key authentication. + # Requires either api_token OR both email and api_key for authentication. dns_cloudflare_api_token: str = "" dns_cloudflare_email: str = "" @@ -53,26 +88,69 @@ class CloudflareProvider(Provider): _validate_aliases = alias_model_validator( { - "dns_cloudflare_api_token": ("dns_cloudflare_api_token", "cloudflare_api_token", "api_token"), - "dns_cloudflare_email": ("dns_cloudflare_email", "cloudflare_email", "email"), - "dns_cloudflare_api_key": ("dns_cloudflare_api_key", "cloudflare_api_key", "api_key"), + "dns_cloudflare_api_token": ( + "dns_cloudflare_api_token", "cloudflare_api_token", "api_token" + ), + "dns_cloudflare_email": ( + "dns_cloudflare_email", "cloudflare_email", "email" + ), + "dns_cloudflare_api_key": ( + "dns_cloudflare_api_key", "cloudflare_api_key", "api_key" + ), } ) + # Return the formatted credentials, excluding defaults. + # Only includes non-empty credential fields to avoid cluttering output. + # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: - """Return the formatted credentials, excluding defaults.""" - return "\n".join(f"{key} = {value}" for key, value in self.model_dump(exclude={"file_type"}, exclude_defaults=True).items()).encode("utf-8") - + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + all_fields = self.model_dump(exclude={"file_type"}) + non_default_fields = self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ) + print(f"DEBUG: Cloudflare provider has {len(all_fields)} total fields") + print(f"DEBUG: {len(non_default_fields)} non-default fields will be included") + + content = "\n".join( + f"{key} = {value}" + for key, value in self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ).items() + ).encode("utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(content)} bytes of Cloudflare credentials") + + return content + + # Validate Cloudflare credentials. + # Ensures either API token or email+API key combination is provided. + # Raises: ValueError if neither authentication method is complete @model_validator(mode="after") def validate_cloudflare_credentials(self): - """Validate Cloudflare credentials.""" - if not self.dns_cloudflare_api_token and not (self.dns_cloudflare_email and self.dns_cloudflare_api_key): - raise ValueError("Either 'dns_cloudflare_api_token' or both 'dns_cloudflare_email' and 'dns_cloudflare_api_key' must be provided.") + has_token = bool(self.dns_cloudflare_api_token) + has_legacy = bool(self.dns_cloudflare_email and self.dns_cloudflare_api_key) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Cloudflare credential validation:") + print(f"DEBUG: API token provided: {has_token}") + print(f"DEBUG: Legacy email+key provided: {has_legacy}") + + if not has_token and not has_legacy: + raise ValueError( + "Either 'dns_cloudflare_api_token' or both " + "'dns_cloudflare_email' and 'dns_cloudflare_api_key' must be provided." + ) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print("DEBUG: Cloudflare credentials validation passed") + return self class DesecProvider(Provider): - """deSEC DNS provider.""" + # Requires only an API token for authentication. dns_desec_token: str @@ -84,19 +162,21 @@ class DesecProvider(Provider): class DigitalOceanProvider(Provider): - """DigitalOcean DNS provider.""" + # Requires a personal access token with read/write scope. dns_digitalocean_token: str _validate_aliases = alias_model_validator( { - "dns_digitalocean_token": ("dns_digitalocean_token", "digitalocean_token", "token"), + "dns_digitalocean_token": ( + "dns_digitalocean_token", "digitalocean_token", "token" + ), } ) class DnsimpleProvider(Provider): - """DNSimple DNS provider.""" + # Requires an API token for authentication. dns_dnsimple_token: str @@ -108,35 +188,45 @@ class DnsimpleProvider(Provider): class DnsMadeEasyProvider(Provider): - """DNS Made Easy DNS provider.""" + # Requires API key and secret key. + # Both keys are required for authentication. dns_dnsmadeeasy_api_key: str dns_dnsmadeeasy_secret_key: str _validate_aliases = alias_model_validator( { - "dns_dnsmadeeasy_api_key": ("dns_dnsmadeeasy_api_key", "dnsmadeeasy_api_key", "api_key"), - "dns_dnsmadeeasy_secret_key": ("dns_dnsmadeeasy_secret_key", "dnsmadeeasy_secret_key", "secret_key"), + "dns_dnsmadeeasy_api_key": ( + "dns_dnsmadeeasy_api_key", "dnsmadeeasy_api_key", "api_key" + ), + "dns_dnsmadeeasy_secret_key": ( + "dns_dnsmadeeasy_secret_key", "dnsmadeeasy_secret_key", "secret_key" + ), } ) class GehirnProvider(Provider): - """Gehirn DNS provider.""" + # Requires both API token and API secret for authentication. dns_gehirn_api_token: str dns_gehirn_api_secret: str _validate_aliases = alias_model_validator( { - "dns_gehirn_api_token": ("dns_gehirn_api_token", "gehirn_api_token", "api_token"), - "dns_gehirn_api_secret": ("dns_gehirn_api_secret", "gehirn_api_secret", "api_secret"), + "dns_gehirn_api_token": ( + "dns_gehirn_api_token", "gehirn_api_token", "api_token" + ), + "dns_gehirn_api_secret": ( + "dns_gehirn_api_secret", "gehirn_api_secret", "api_secret" + ), } ) class GoogleProvider(Provider): - """Google Cloud DNS provider.""" + # Uses Google Cloud service account credentials in JSON format. + # Requires a service account with DNS admin permissions. type: str = "service_account" project_id: str @@ -153,41 +243,71 @@ class GoogleProvider(Provider): { "type": ("type", "google_type", "dns_google_type"), "project_id": ("project_id", "google_project_id", "dns_google_project_id"), - "private_key_id": ("private_key_id", "google_private_key_id", "dns_google_private_key_id"), - "private_key": ("private_key", "google_private_key", "dns_google_private_key"), - "client_email": ("client_email", "google_client_email", "dns_google_client_email"), + "private_key_id": ( + "private_key_id", "google_private_key_id", "dns_google_private_key_id" + ), + "private_key": ( + "private_key", "google_private_key", "dns_google_private_key" + ), + "client_email": ( + "client_email", "google_client_email", "dns_google_client_email" + ), "client_id": ("client_id", "google_client_id", "dns_google_client_id"), "auth_uri": ("auth_uri", "google_auth_uri", "dns_google_auth_uri"), "token_uri": ("token_uri", "google_token_uri", "dns_google_token_uri"), - "auth_provider_x509_cert_url": ("auth_provider_x509_cert_url", "google_auth_provider_x509_cert_url", "dns_google_auth_provider_x509_cert_url"), - "client_x509_cert_url": ("client_x509_cert_url", "google_client_x509_cert_url", "dns_google_client_x509_cert_url"), + "auth_provider_x509_cert_url": ( + "auth_provider_x509_cert_url", + "google_auth_provider_x509_cert_url", + "dns_google_auth_provider_x509_cert_url" + ), + "client_x509_cert_url": ( + "client_x509_cert_url", + "google_client_x509_cert_url", + "dns_google_client_x509_cert_url" + ), } ) + # Return the formatted credentials in JSON format. + # Google Cloud requires credentials in JSON service account format. + # Returns: bytes - UTF-8 encoded JSON content def get_formatted_credentials(self) -> bytes: - """Return the formatted credentials in JSON format.""" - return self.model_dump_json(indent=2, exclude={"file_type"}).encode("utf-8") - + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print("DEBUG: Google provider formatting credentials as JSON") + + json_content = self.model_dump_json( + indent=2, exclude={"file_type"} + ).encode("utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(json_content)} bytes of JSON credentials") + + return json_content + + # Return the file type that the credentials should be written to. + # Google provider requires JSON format for service account credentials. + # Returns: Literal type indicating JSON file extension @staticmethod def get_file_type() -> Literal["json"]: - """Return the file type that the credentials should be written to.""" return "json" class InfomaniakProvider(Provider): - """Infomaniak DNS provider.""" + # Requires an API token for authentication. dns_infomaniak_token: str _validate_aliases = alias_model_validator( { - "dns_infomaniak_token": ("dns_infomaniak_token", "infomaniak_token", "token"), + "dns_infomaniak_token": ( + "dns_infomaniak_token", "infomaniak_token", "token" + ), } ) class IonosProvider(Provider): - """Ionos DNS provider.""" + # Requires prefix and secret for authentication, with configurable endpoint. dns_ionos_prefix: str dns_ionos_secret: str @@ -203,7 +323,7 @@ class IonosProvider(Provider): class LinodeProvider(Provider): - """Linode DNS provider.""" + # Requires an API key for authentication. dns_linode_key: str @@ -215,7 +335,8 @@ class LinodeProvider(Provider): class LuaDnsProvider(Provider): - """LuaDns DNS provider.""" + # Requires email and token authentication. + # Both email and token are required for API access. dns_luadns_email: str dns_luadns_token: str @@ -229,7 +350,7 @@ class LuaDnsProvider(Provider): class NSOneProvider(Provider): - """NS1 DNS provider.""" + # Requires an API key for authentication. dns_nsone_api_key: str @@ -241,7 +362,7 @@ class NSOneProvider(Provider): class OvhProvider(Provider): - """OVH DNS provider.""" + # Requires application key, secret, and consumer key for authentication. dns_ovh_endpoint: str = "ovh-eu" dns_ovh_application_key: str @@ -251,15 +372,23 @@ class OvhProvider(Provider): _validate_aliases = alias_model_validator( { "dns_ovh_endpoint": ("dns_ovh_endpoint", "ovh_endpoint", "endpoint"), - "dns_ovh_application_key": ("dns_ovh_application_key", "ovh_application_key", "application_key"), - "dns_ovh_application_secret": ("dns_ovh_application_secret", "ovh_application_secret", "application_secret"), - "dns_ovh_consumer_key": ("dns_ovh_consumer_key", "ovh_consumer_key", "consumer_key"), + "dns_ovh_application_key": ( + "dns_ovh_application_key", "ovh_application_key", "application_key" + ), + "dns_ovh_application_secret": ( + "dns_ovh_application_secret", "ovh_application_secret", + "application_secret" + ), + "dns_ovh_consumer_key": ( + "dns_ovh_consumer_key", "ovh_consumer_key", "consumer_key" + ), } ) class Rfc2136Provider(Provider): - """RFC 2136 DNS provider.""" + # Standard protocol for dynamic DNS updates using TSIG authentication. + # Supports HMAC-based authentication with configurable algorithms. dns_rfc2136_server: str dns_rfc2136_port: Optional[str] = None @@ -274,178 +403,294 @@ class Rfc2136Provider(Provider): "dns_rfc2136_port": ("dns_rfc2136_port", "rfc2136_port", "port"), "dns_rfc2136_name": ("dns_rfc2136_name", "rfc2136_name", "name"), "dns_rfc2136_secret": ("dns_rfc2136_secret", "rfc2136_secret", "secret"), - "dns_rfc2136_algorithm": ("dns_rfc2136_algorithm", "rfc2136_algorithm", "algorithm"), - "dns_rfc2136_sign_query": ("dns_rfc2136_sign_query", "rfc2136_sign_query", "sign_query"), + "dns_rfc2136_algorithm": ( + "dns_rfc2136_algorithm", "rfc2136_algorithm", "algorithm" + ), + "dns_rfc2136_sign_query": ( + "dns_rfc2136_sign_query", "rfc2136_sign_query", "sign_query" + ), } ) + # Return the formatted credentials, excluding defaults. + # RFC2136 provider excludes default values to minimize configuration. + # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: - """Return the formatted credentials, excluding defaults.""" - return "\n".join(f"{key} = {value}" for key, value in self.model_dump(exclude={"file_type"}, exclude_defaults=True).items()).encode("utf-8") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + all_fields = self.model_dump(exclude={"file_type"}) + non_default_fields = self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ) + print(f"DEBUG: RFC2136 provider has {len(all_fields)} total fields") + print(f"DEBUG: {len(non_default_fields)} non-default fields included") + + content = "\n".join( + f"{key} = {value}" + for key, value in self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ).items() + ).encode("utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(content)} bytes of RFC2136 credentials") + + return content class Route53Provider(Provider): - """AWS Route 53 DNS provider.""" + # Uses IAM credentials. + # Requires AWS access key ID and secret access key. aws_access_key_id: str aws_secret_access_key: str _validate_aliases = alias_model_validator( { - "aws_access_key_id": ("aws_access_key_id", "dns_aws_access_key_id", "access_key_id"), - "aws_secret_access_key": ("aws_secret_access_key", "dns_aws_secret_access_key", "secret_access_key"), + "aws_access_key_id": ( + "aws_access_key_id", "dns_aws_access_key_id", "access_key_id" + ), + "aws_secret_access_key": ( + "aws_secret_access_key", "dns_aws_secret_access_key", + "secret_access_key" + ), } ) + # Return the formatted credentials in environment variable format. + # Route53 uses environment variables for AWS credentials. + # Returns: bytes - UTF-8 encoded environment variable format def get_formatted_credentials(self) -> bytes: - """Return the formatted credentials in environment variable format.""" - return "\n".join(f"{key.upper()}={value!r}" for key, value in self.model_dump(exclude={"file_type"}).items()).encode("utf-8") - + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + fields = self.model_dump(exclude={"file_type"}) + print(f"DEBUG: Route53 provider formatting {len(fields)} fields as env vars") + + content = "\n".join( + f"{key.upper()}={value!r}" + for key, value in self.model_dump(exclude={"file_type"}).items() + ).encode("utf-8") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(content)} bytes of environment variables") + + return content + + # Return the file type that the credentials should be written to. + # Route53 provider uses environment variable format. + # Returns: Literal type indicating env file extension @staticmethod def get_file_type() -> Literal["env"]: - """Return the file type that the credentials should be written to.""" return "env" class SakuraCloudProvider(Provider): - """Sakura Cloud DNS provider.""" + # Requires API token and secret for authentication. dns_sakuracloud_api_token: str dns_sakuracloud_api_secret: str _validate_aliases = alias_model_validator( { - "dns_sakuracloud_api_token": ("dns_sakuracloud_api_token", "sakuracloud_api_token", "api_token"), - "dns_sakuracloud_api_secret": ("dns_sakuracloud_api_secret", "sakuracloud_api_secret", "api_secret"), + "dns_sakuracloud_api_token": ( + "dns_sakuracloud_api_token", "sakuracloud_api_token", "api_token" + ), + "dns_sakuracloud_api_secret": ( + "dns_sakuracloud_api_secret", "sakuracloud_api_secret", "api_secret" + ), } ) class ScalewayProvider(Provider): - """Scaleway DNS provider.""" + # Requires an application token for authentication. dns_scaleway_application_token: str _validate_aliases = alias_model_validator( { - "dns_scaleway_application_token": ("dns_scaleway_application_token", "scaleway_application_token", "application_token"), + "dns_scaleway_application_token": ( + "dns_scaleway_application_token", "scaleway_application_token", + "application_token" + ), } ) class NjallaProvider(Provider): - """Njalla DNS provider.""" + # Requires an API token for authentication. dns_njalla_token: str _validate_aliases = alias_model_validator( { - "dns_njalla_token": ("dns_njalla_token", "njalla_token", "token", "api_token", "auth_token"), + "dns_njalla_token": ( + "dns_njalla_token", "njalla_token", "token", "api_token", "auth_token" + ), } ) class WildcardGenerator: - """Manages the generation of wildcard domains across domain groups.""" + # Manages the generation of wildcard domains across domain groups. + # Handles grouping of domains and automatic wildcard pattern generation + # for efficient certificate management across multiple subdomains. def __init__(self): - self.__domain_groups = {} # Stores raw domains grouped by identifier - self.__wildcards = {} # Stores generated wildcard patterns - + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print("DEBUG: Initializing WildcardGenerator") + + # Stores raw domains grouped by identifier + self.__domain_groups = {} + # Stores generated wildcard patterns + self.__wildcards = {} + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print("DEBUG: WildcardGenerator initialized with empty groups") + + # Add domains to a group and regenerate wildcards. + # Organizes domains by group and environment for wildcard generation. + # Args: group - Group identifier for these domains + # domains - List of domains to add + # email - Contact email for this domain group + # staging - Whether these domains are for staging environment def extend(self, group: str, domains: List[str], email: str, staging: bool = False): - """ - Add domains to a group and regenerate wildcards. - - Args: - group: Group identifier for these domains - domains: List of domains to add - email: Contact email for this domain group - staging: Whether these domains are for staging environment - """ + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Extending group '{group}' with {len(domains)} domains") + print(f"DEBUG: Staging mode: {staging}, Email: {email}") + print(f"DEBUG: Domains: {domains}") + # Initialize group if it doesn't exist if group not in self.__domain_groups: - self.__domain_groups[group] = {"staging": set(), "prod": set(), "email": email} + self.__domain_groups[group] = { + "staging": set(), + "prod": set(), + "email": email + } + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Created new domain group '{group}'") # Add domains to appropriate environment env_type = "staging" if staging else "prod" + domains_added = 0 for domain in domains: if domain := domain.strip(): self.__domain_groups[group][env_type].add(domain) + domains_added += 1 + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Added {domains_added} domains to {env_type} environment") + total_staging = len(self.__domain_groups[group]["staging"]) + total_prod = len(self.__domain_groups[group]["prod"]) + print(f"DEBUG: Group '{group}' now has {total_staging} staging, {total_prod} prod domains") # Regenerate wildcards after adding new domains self.__generate_wildcards(staging) + # Generate wildcard patterns for the specified environment. + # Creates optimized wildcard certificates that cover multiple subdomains. + # Args: staging - Whether to generate wildcards for staging environment def __generate_wildcards(self, staging: bool = False): - """ - Generate wildcard patterns for the specified environment. - - Args: - staging: Whether to generate wildcards for staging environment - """ + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + env_type = "staging" if staging else "prod" + print(f"DEBUG: Generating wildcards for {env_type} environment") + print(f"DEBUG: Processing {len(self.__domain_groups)} domain groups") + self.__wildcards.clear() env_type = "staging" if staging else "prod" + wildcards_generated = 0 # Process each domain group for group, types in self.__domain_groups.items(): if group not in self.__wildcards: - self.__wildcards[group] = {"staging": set(), "prod": set(), "email": types["email"]} + self.__wildcards[group] = { + "staging": set(), + "prod": set(), + "email": types["email"] + } # Process each domain in the group for domain in types[env_type]: # Convert domain to wildcards and add to appropriate group self.__add_domain_wildcards(domain, group, env_type) + wildcards_generated += 1 + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated wildcards for {wildcards_generated} domains") + + # Convert a domain to wildcard patterns and add to the wildcards collection. + # Determines optimal wildcard patterns based on domain structure. + # Args: domain - Domain to process + # group - Group identifier + # env_type - Environment type (staging or prod) def __add_domain_wildcards(self, domain: str, group: str, env_type: str): - """ - Convert a domain to wildcard patterns and add to the wildcards collection. - - Args: - domain: Domain to process - group: Group identifier - env_type: Environment type (staging or prod) - """ + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Processing domain '{domain}' for wildcards") + parts = domain.split(".") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Domain has {len(parts)} parts: {parts}") # Handle subdomains (domains with more than 2 parts) if len(parts) > 2: # Create wildcard for the base domain (e.g., *.example.com) base_domain = ".".join(parts[1:]) - self.__wildcards[group][env_type].add(f"*.{base_domain}") + wildcard_domain = f"*.{base_domain}" + + self.__wildcards[group][env_type].add(wildcard_domain) self.__wildcards[group][env_type].add(base_domain) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Added wildcard '{wildcard_domain}' and base '{base_domain}'") else: # Just add the raw domain for top-level domains self.__wildcards[group][env_type].add(domain) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Added top-level domain '{domain}' directly") + # Get formatted wildcard domains for each group. + # Returns organized wildcard data ready for certificate generation. + # Returns: Dictionary of group data with formatted wildcard domains def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", "email"], str]]: - """ - Get formatted wildcard domains for each group. - - Returns: - Dictionary of group data with formatted wildcard domains - """ + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Formatting wildcards for {len(self.__wildcards)} groups") + result = {} + total_domains = 0 + for group, data in self.__wildcards.items(): result[group] = {"email": data["email"]} + for env_type in ("staging", "prod"): if domains := data[env_type]: # Sort domains with wildcards first - result[group][env_type] = ",".join(sorted(domains, key=lambda x: x[0] != "*")) + sorted_domains = sorted(domains, key=lambda x: x[0] != "*") + result[group][env_type] = ",".join(sorted_domains) + total_domains += len(domains) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Group '{group}' {env_type}: {len(domains)} domains") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Formatted {total_domains} total wildcard domains") + return result + # Generate wildcard patterns from a list of domains. + # Static method for generating wildcards without managing groups. + # Args: domains - List of domains to process + # Returns: List of extracted wildcard domains @staticmethod def extract_wildcards_from_domains(domains: List[str]) -> List[str]: - """ - Generate wildcard patterns from a list of domains. - - Args: - domains: List of domains to process - - Returns: - List of extracted wildcard domains - """ + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Extracting wildcards from {len(domains)} domains") + print(f"DEBUG: Input domains: {domains}") + wildcards = set() + for domain in domains: parts = domain.split(".") + # Generate wildcards for subdomains if len(parts) > 2: base_domain = ".".join(parts[1:]) @@ -456,37 +701,46 @@ def extract_wildcards_from_domains(domains: List[str]) -> List[str]: wildcards.add(domain) # Sort with wildcards first - return sorted(wildcards, key=lambda x: x[0] != "*") + result = sorted(wildcards, key=lambda x: x[0] != "*") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated {len(result)} wildcard patterns: {result}") + + return result + # Extract the base domain from a domain name. + # Removes wildcard prefix if present to get the actual domain. + # Args: domain - Input domain name + # Returns: Base domain (without wildcard prefix if present) @staticmethod def get_base_domain(domain: str) -> str: - """ - Extract the base domain from a domain name. - - Args: - domain: Input domain name - - Returns: - Base domain (without wildcard prefix if present) - """ - return domain.lstrip("*.") - + base = domain.lstrip("*.") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + if domain != base: + print(f"DEBUG: Extracted base domain '{base}' from '{domain}'") + + return base + + # Generate a consistent group name for wildcards. + # Creates a unique identifier for grouping related wildcard certificates. + # Args: domain - The domain name + # provider - DNS provider name or 'http' for HTTP challenge + # challenge_type - Challenge type (dns or http) + # staging - Whether this is for staging environment + # content_hash - Hash of credential content + # profile - Certificate profile (classic, tlsserver or shortlived) + # Returns: A formatted group name string @staticmethod - def create_group_name(domain: str, provider: str, challenge_type: str, staging: bool, content_hash: str, profile: str = "classic") -> str: - """ - Generate a consistent group name for wildcards. - - Args: - domain: The domain name - provider: DNS provider name or 'http' for HTTP challenge - challenge_type: Challenge type (dns or http) - staging: Whether this is for staging environment - content_hash: Hash of credential content - profile: Certificate profile (classic, tlsserver or shortlived) - - Returns: - A formatted group name string - """ + def create_group_name(domain: str, provider: str, challenge_type: str, + staging: bool, content_hash: str, + profile: str = "classic") -> str: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Creating group name for domain '{domain}'") + print(f"DEBUG: Provider: {provider}, Challenge: {challenge_type}") + print(f"DEBUG: Staging: {staging}, Profile: {profile}") + print(f"DEBUG: Content hash: {content_hash[:10]}...") + # Extract base domain and format it for the group name base_domain = WildcardGenerator.get_base_domain(domain).replace(".", "-") env = "staging" if staging else "prod" @@ -494,4 +748,9 @@ def create_group_name(domain: str, provider: str, challenge_type: str, staging: # Use provider name for DNS challenge, otherwise use 'http' challenge_identifier = provider if challenge_type == "dns" else "http" - return f"{challenge_identifier}_{env}_{profile}_{base_domain}_{content_hash}" + group_name = f"{challenge_identifier}_{env}_{profile}_{base_domain}_{content_hash}" + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Generated group name: '{group_name}'") + + return group_name \ No newline at end of file From acc54af9ed75db19f6a4ea29cb63b174714aa727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 08:48:21 +0200 Subject: [PATCH 04/15] update formating, documentation of functions, add more debugging output - formating: apply 79-character line limits for code (120 max fallback) - documentation: aff documentation of functions - debugging: add debugging output when LOG_LEVEL: "DEBUG" --- .../core/letsencrypt/jobs/certbot-auth.py | 69 +- .../core/letsencrypt/jobs/certbot-cleanup.py | 65 +- .../core/letsencrypt/jobs/certbot-deploy.py | 82 ++- .../core/letsencrypt/jobs/certbot-new.py | 660 +++++++++++++++--- .../core/letsencrypt/jobs/certbot-renew.py | 131 +++- .../core/letsencrypt/jobs/letsencrypt.py | 326 ++++++--- 6 files changed, 1032 insertions(+), 301 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-auth.py b/src/common/core/letsencrypt/jobs/certbot-auth.py index 559d2da2d4..c3b28b5afd 100644 --- a/src/common/core/letsencrypt/jobs/certbot-auth.py +++ b/src/common/core/letsencrypt/jobs/certbot-auth.py @@ -7,7 +7,8 @@ from traceback import format_exc for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) - for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: + for paths in (("deps", "python"), ("utils",), ("api",), + ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -27,9 +28,12 @@ validation = getenv("CERTBOT_VALIDATION", "") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ACME HTTP challenge authentication started") + LOGGER.debug("ACME HTTP challenge authentication started") LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") LOGGER.debug(f"Validation length: {len(validation) if validation else 0} chars") + LOGGER.debug("Checking for required environment variables") + LOGGER.debug(f"CERTBOT_TOKEN exists: {bool(token)}") + LOGGER.debug(f"CERTBOT_VALIDATION exists: {bool(validation)}") # Detect the current BunkerWeb integration type # This determines how we handle the challenge deployment @@ -37,6 +41,7 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Integration detection completed: {integration}") + LOGGER.debug("Determining challenge deployment method based on integration") LOGGER.info(f"Detected {integration} integration") @@ -46,6 +51,7 @@ if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Cluster integration detected, initializing database connection") + LOGGER.debug("Will distribute challenge to all cluster instances via API") # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -55,11 +61,13 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Retrieved {len(instances)} instances from database") + LOGGER.debug("Instance details:") for i, instance in enumerate(instances): LOGGER.debug( - f"Instance {i+1}: {instance['hostname']}:" + f" Instance {i+1}: {instance['hostname']}:" f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" ) + LOGGER.debug("Preparing to send challenge data to each instance") LOGGER.info(f"Sending challenge to {len(instances)} instances") @@ -67,9 +75,10 @@ for instance in instances: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( - f"Sending challenge to instance: " - f"{instance['hostname']}:{instance['port']}" + f"Processing instance: {instance['hostname']}:{instance['port']}" ) + LOGGER.debug(f"Server name: {instance.get('server_name', 'N/A')}") + LOGGER.debug("Creating API client for this instance") # Create API client for this instance api = API( @@ -77,6 +86,12 @@ host=instance["server_name"] ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API endpoint: {api.endpoint}") + LOGGER.debug("Preparing challenge data payload") + LOGGER.debug(f"Token: {token[:10]}... (truncated)") + LOGGER.debug(f"Validation length: {len(validation)} characters") + # Send POST request to deploy the challenge sent, err, status_code, resp = api.request( "POST", @@ -92,6 +107,7 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"API request failed with error: {err}") + LOGGER.debug("This instance will not receive the challenge") elif status_code != 200: status = 1 LOGGER.error( @@ -100,27 +116,27 @@ f"status = {resp['status']}, msg = {resp['msg']}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API response details: {resp}") + LOGGER.debug(f"HTTP status code: {status_code}") + LOGGER.debug(f"Response details: {resp}") + LOGGER.debug("Challenge deployment failed for this instance") else: LOGGER.info( f"Successfully sent API request to " f"{api.endpoint}/lets-encrypt/challenge" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Challenge successfully deployed to " - f"{instance['hostname']}:{instance['port']}" - ) + LOGGER.debug(f"HTTP status code: {status_code}") + LOGGER.debug("Challenge successfully deployed to instance") + LOGGER.debug(f"Instance can now serve challenge at /.well-known/acme-challenge/{token}") # Linux case: Standalone installation # For standalone Linux installations, we write the challenge # file directly to the local filesystem else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - "Standalone Linux integration detected, " - "writing challenge file locally" - ) + LOGGER.debug("Standalone Linux integration detected") + LOGGER.debug("Writing challenge file directly to local filesystem") + LOGGER.debug("No API distribution needed for standalone mode") # Create the ACME challenge directory structure # This follows the standard .well-known/acme-challenge path @@ -130,22 +146,34 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Creating challenge directory: {root_dir}") + LOGGER.debug(f"Challenge directory path: {root_dir}") + LOGGER.debug("Creating directory structure if it doesn't exist") + LOGGER.debug("Directory will be created with parents=True, exist_ok=True") # Create directory structure with appropriate permissions root_dir.mkdir(parents=True, exist_ok=True) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Directory structure created successfully") + LOGGER.debug(f"Directory exists: {root_dir.exists()}") + LOGGER.debug(f"Directory is writable: {root_dir.is_dir()}") + # Write the challenge validation content to the token file challenge_file = root_dir.joinpath(token) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Writing challenge file: {challenge_file}") - LOGGER.debug(f"Challenge file content length: {len(validation)} bytes") + LOGGER.debug(f"Challenge file path: {challenge_file}") + LOGGER.debug(f"Token filename: {token}") + LOGGER.debug(f"Validation content length: {len(validation)} bytes") + LOGGER.debug("Writing validation content to challenge file") challenge_file.write_text(validation, encoding="utf-8") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Challenge file written successfully") + LOGGER.debug(f"File exists: {challenge_file.exists()}") + LOGGER.debug(f"File size: {challenge_file.stat().st_size} bytes") + LOGGER.debug("Let's Encrypt can now access the challenge file") LOGGER.info(f"Challenge file created at {challenge_file}") @@ -156,8 +184,15 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Full exception traceback logged above") + LOGGER.debug("Authentication process failed due to exception") + LOGGER.debug("Let's Encrypt challenge will not be available") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"ACME HTTP challenge authentication completed with status: {status}") + if status == 0: + LOGGER.debug("Authentication completed successfully") + LOGGER.debug("Let's Encrypt can now access the challenge") + else: + LOGGER.debug("Authentication failed - challenge may not be accessible") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-cleanup.py b/src/common/core/letsencrypt/jobs/certbot-cleanup.py index 6509fdd212..3f6716dbff 100644 --- a/src/common/core/letsencrypt/jobs/certbot-cleanup.py +++ b/src/common/core/letsencrypt/jobs/certbot-cleanup.py @@ -7,7 +7,8 @@ from traceback import format_exc for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) - for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: + for paths in (("deps", "python"), ("utils",), ("api",), + ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -25,8 +26,10 @@ token = getenv("CERTBOT_TOKEN", "") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ACME HTTP challenge cleanup started") + LOGGER.debug("ACME HTTP challenge cleanup started") LOGGER.debug(f"Token to clean: {token[:10] if token else 'None'}...") + LOGGER.debug("Starting cleanup process for Let's Encrypt challenge") + LOGGER.debug("This will remove challenge files from all instances") # Detect the current BunkerWeb integration type # This determines how we handle the challenge cleanup process @@ -34,6 +37,7 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Integration detection completed: {integration}") + LOGGER.debug("Determining cleanup method based on integration type") LOGGER.info(f"Detected {integration} integration") @@ -42,9 +46,9 @@ # from all instances via the BunkerWeb API if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - "Cluster integration detected, initializing database connection" - ) + LOGGER.debug("Cluster integration detected") + LOGGER.debug("Will remove challenge from all cluster instances via API") + LOGGER.debug("Initializing database connection to get instance list") # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -54,11 +58,13 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Retrieved {len(instances)} instances from database") + LOGGER.debug("Instance details for cleanup:") for i, instance in enumerate(instances): LOGGER.debug( - f"Instance {i+1}: {instance['hostname']}:" + f" Instance {i+1}: {instance['hostname']}:" f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" ) + LOGGER.debug("Preparing to send DELETE requests to each instance") LOGGER.info(f"Cleaning challenge from {len(instances)} instances") @@ -66,9 +72,11 @@ for instance in instances: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( - f"Cleaning challenge from instance: " + f"Processing cleanup for instance: " f"{instance['hostname']}:{instance['port']}" ) + LOGGER.debug(f"Server name: {instance.get('server_name', 'N/A')}") + LOGGER.debug("Creating API client for cleanup request") # Create API client for this instance api = API( @@ -76,6 +84,11 @@ host=instance["server_name"] ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"API endpoint: {api.endpoint}") + LOGGER.debug("Preparing DELETE request for challenge cleanup") + LOGGER.debug(f"Token to delete: {token}") + # Send DELETE request to remove the challenge sent, err, status_code, resp = api.request( "DELETE", @@ -90,7 +103,8 @@ f"{api.endpoint}/lets-encrypt/challenge: {err}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API request failed with error: {err}") + LOGGER.debug(f"DELETE request failed with error: {err}") + LOGGER.debug("Challenge file may remain on this instance") elif status_code != 200: status = 1 LOGGER.error( @@ -99,27 +113,27 @@ f"status = {resp['status']}, msg = {resp['msg']}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API response details: {resp}") + LOGGER.debug(f"HTTP status code: {status_code}") + LOGGER.debug(f"Response details: {resp}") + LOGGER.debug("Challenge cleanup failed for this instance") else: LOGGER.info( f"Successfully sent API request to " f"{api.endpoint}/lets-encrypt/challenge" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Challenge successfully cleaned from " - f"{instance['hostname']}:{instance['port']}" - ) + LOGGER.debug(f"HTTP status code: {status_code}") + LOGGER.debug("Challenge successfully removed from instance") + LOGGER.debug(f"Token {token} has been cleaned up") # Linux case: Standalone installation # For standalone Linux installations, we remove the challenge # file directly from the local filesystem else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - "Standalone Linux integration detected, " - "removing challenge file locally" - ) + LOGGER.debug("Standalone Linux integration detected") + LOGGER.debug("Removing challenge file directly from local filesystem") + LOGGER.debug("No API cleanup needed for standalone mode") # Construct path to the ACME challenge file # This follows the standard .well-known/acme-challenge path @@ -129,16 +143,22 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Challenge file to remove: {challenge_file}") + LOGGER.debug(f"Challenge file path: {challenge_file}") + LOGGER.debug(f"Token filename: {token}") + LOGGER.debug("Checking if challenge file exists before cleanup") LOGGER.debug(f"File exists before cleanup: {challenge_file.exists()}") + if challenge_file.exists(): + LOGGER.debug(f"File size: {challenge_file.stat().st_size} bytes") # Remove the challenge file if it exists # missing_ok=True prevents errors if file doesn't exist challenge_file.unlink(missing_ok=True) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Challenge file unlink operation completed") LOGGER.debug(f"File exists after cleanup: {challenge_file.exists()}") - LOGGER.debug("Local challenge file cleanup completed") + LOGGER.debug("Local challenge file cleanup completed successfully") + LOGGER.debug("Challenge is no longer accessible to Let's Encrypt") LOGGER.info(f"Challenge file removed: {challenge_file}") @@ -149,8 +169,15 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Full exception traceback logged above") + LOGGER.debug("Cleanup process failed due to exception") + LOGGER.debug("Some challenge files may remain uncleaned") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"ACME HTTP challenge cleanup completed with status: {status}") + if status == 0: + LOGGER.debug("Cleanup completed successfully") + LOGGER.debug("All challenge files have been removed") + else: + LOGGER.debug("Cleanup encountered errors - some files may remain") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index ba56d3ffad..b0ff874c0e 100644 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -8,7 +8,8 @@ from traceback import format_exc for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) - for paths in (("deps", "python"), ("utils",), ("api",), ("db",))]: + for paths in (("deps", "python"), ("utils",), ("api",), + ("db",))]: if deps_path not in sys_path: sys_path.append(deps_path) @@ -27,9 +28,11 @@ renewed_domains = getenv("RENEWED_DOMAINS", "") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate deployment started") + LOGGER.debug("Certificate deployment started") LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") LOGGER.debug(f"Renewed domains: {renewed_domains}") + LOGGER.debug("Starting certificate distribution to cluster instances") + LOGGER.debug("This process will update certificates on all instances") LOGGER.info(f"Certificates renewal for {renewed_domains} successful") @@ -38,12 +41,18 @@ # for efficient transfer to cluster instances if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Creating certificate archive for distribution") + LOGGER.debug("Packaging all Let's Encrypt certificates into tarball") + LOGGER.debug("Archive will contain fullchain.pem and privkey.pem files") tgz = BytesIO() - cert_source_path = join(sep, "var", "cache", "bunkerweb", "letsencrypt", "etc") + cert_source_path = join(sep, "var", "cache", "bunkerweb", "letsencrypt", + "etc") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Source certificate path: {cert_source_path}") + LOGGER.debug("Checking if certificate directory exists") + cert_path_exists = Path(cert_source_path).exists() + LOGGER.debug(f"Certificate directory exists: {cert_path_exists}") # Create compressed tarball containing certificate files # compresslevel=3 provides good compression with reasonable performance @@ -55,11 +64,16 @@ files = {"archive.tar.gz": tgz} if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate archive created, size: {tgz.getbuffer().nbytes} bytes") + archive_size = tgz.getbuffer().nbytes + LOGGER.debug(f"Certificate archive created successfully") + LOGGER.debug(f"Archive size: {archive_size} bytes") + LOGGER.debug(f"Compression level: 3 (balanced speed/size)") + LOGGER.debug("Archive ready for distribution to instances") # Initialize database connection to get cluster instances if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Initializing database connection") + LOGGER.debug("Need to get list of active BunkerWeb instances") db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -78,11 +92,13 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Retrieved {len(instances)} instances from database") LOGGER.debug(f"Found {len(services)} configured services: {services}") + LOGGER.debug("Instance details for certificate deployment:") for i, instance in enumerate(instances): LOGGER.debug( - f"Instance {i+1}: {instance['hostname']}:" + f" Instance {i+1}: {instance['hostname']}:" f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" ) + LOGGER.debug("Will deploy certificates and trigger reload on each instance") # Configure reload timeout based on environment and service count # Minimum timeout prevents premature timeouts on slow systems @@ -98,10 +114,11 @@ calculated_timeout = max(reload_min_timeout, 3 * len(services)) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Reload timeout configuration:") - LOGGER.debug(f" Minimum timeout: {reload_min_timeout}s") - LOGGER.debug(f" Service count: {len(services)}") - LOGGER.debug(f" Calculated timeout: {calculated_timeout}s") + LOGGER.debug("Reload timeout configuration:") + LOGGER.debug(f" Minimum timeout setting: {reload_min_timeout}s") + LOGGER.debug(f" Number of services: {len(services)}") + LOGGER.debug(f" Calculated timeout (max of min or 3s per service): {calculated_timeout}s") + LOGGER.debug("Timeout ensures all services have time to reload certificates") LOGGER.info(f"Deploying certificates to {len(instances)} instances") @@ -109,19 +126,24 @@ for i, instance in enumerate(instances): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Processing instance {i+1}/{len(instances)}") + LOGGER.debug(f"Current instance: {instance['hostname']}:{instance['port']}") # Construct API endpoint for this instance endpoint = f"http://{instance['hostname']}:{instance['port']}" host = instance["server_name"] if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Connecting to: {endpoint} (host: {host})") + LOGGER.debug(f"API endpoint: {endpoint}") + LOGGER.debug(f"Host header: {host}") + LOGGER.debug("Creating API client for certificate upload") api = API(endpoint, host=host) # Upload certificate archive to the instance if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Uploading certificate archive to {endpoint}") + LOGGER.debug("Sending POST request with certificate tarball") + LOGGER.debug("This will extract certificates to instance filesystem") sent, err, status_code, resp = api.request( "POST", @@ -137,6 +159,7 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Certificate upload failed with error: {err}") + LOGGER.debug("Skipping reload for this instance due to upload failure") continue elif status_code != 200: status = 1 @@ -146,7 +169,9 @@ f"status = {resp['status']}, msg = {resp['msg']}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate upload response: {resp}") + LOGGER.debug(f"HTTP status code: {status_code}") + LOGGER.debug(f"Error response: {resp}") + LOGGER.debug("Certificate upload failed, skipping reload") continue else: LOGGER.info( @@ -154,17 +179,21 @@ f"{api.endpoint}/lets-encrypt/certificates" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate archive uploaded successfully") + LOGGER.debug("Certificate archive uploaded successfully") + LOGGER.debug("Certificates are now available on instance filesystem") + LOGGER.debug("Proceeding to trigger configuration reload") # Trigger configuration reload on the instance # Configuration testing can be disabled via environment variable - disable_testing = getenv("DISABLE_CONFIGURATION_TESTING", "no").lower() + disable_testing = getenv("DISABLE_CONFIGURATION_TESTING", + "no").lower() test_config = "no" if disable_testing == "yes" else "yes" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Triggering reload on {endpoint}") - LOGGER.debug(f"Configuration testing: {test_config}") + LOGGER.debug(f"Triggering configuration reload on {endpoint}") + LOGGER.debug(f"Configuration testing enabled: {test_config}") LOGGER.debug(f"Reload timeout: {calculated_timeout}s") + LOGGER.debug("This will reload nginx with new certificates") sent, err, status_code, resp = api.request( "POST", @@ -179,6 +208,7 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Reload request failed with error: {err}") + LOGGER.debug("Instance may not have reloaded new certificates") elif status_code != 200: status = 1 LOGGER.error( @@ -186,19 +216,29 @@ f"status = {resp['status']}, msg = {resp['msg']}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"HTTP status code: {status_code}") LOGGER.debug(f"Reload response: {resp}") + LOGGER.debug("Configuration reload failed on this instance") else: LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Instance {endpoint} reloaded successfully") + LOGGER.debug("Configuration reload completed successfully") + LOGGER.debug("New certificates are now active on this instance") + LOGGER.debug(f"Instance {endpoint} fully updated") # Reset file pointer for next instance if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Resetting archive buffer for next instance") + LOGGER.debug("Resetting archive buffer position for next instance") + LOGGER.debug("Archive will be re-read from beginning for next upload") tgz.seek(0, 0) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Certificate deployment process completed") + LOGGER.debug("All instances have been processed") + if status == 0: + LOGGER.debug("All deployments successful") + else: + LOGGER.debug("Some deployments failed - check individual instance logs") except BaseException as e: status = 1 @@ -207,8 +247,14 @@ if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Full exception traceback logged above") + LOGGER.debug("Certificate deployment failed due to exception") + LOGGER.debug("Some instances may not have received updated certificates") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate deployment completed with status: {status}") + LOGGER.debug(f"Certificate deployment completed with final status: {status}") + if status == 0: + LOGGER.debug("All certificates deployed successfully across cluster") + else: + LOGGER.debug("Deployment completed with errors - manual intervention may be needed") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 3c1abb15ec..75133d1f16 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -80,14 +80,19 @@ ) -# Load and cache the public suffix list for domain validation. -# Fetches the PSL from the official source and caches it locally. -# Returns cached version if available and fresh (less than 1 day old). -# Args: job - Job instance for caching operations -# Returns: list - Lines from the public suffix list file def load_public_suffix_list(job): + # Load and cache the public suffix list for domain validation. + # Fetches the PSL from the official source and caches it locally. + # Returns cached version if available and fresh (less than 1 day old). + # + # Args: + # job - Job instance for caching operations + # + # Returns: + # list - Lines from the public suffix list file if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Loading public suffix list from cache or {PSL_URL}") + LOGGER.debug("Checking if cached PSL is available and fresh") job_cache = job.get_cache(PSL_STATIC_FILE, with_info=True, with_data=True) if ( @@ -99,41 +104,74 @@ def load_public_suffix_list(job): ): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Using cached public suffix list") + LOGGER.debug(f"Cache age: {(datetime.now().astimezone().timestamp() - job_cache['last_update']) / 3600:.1f} hours") return job_cache["data"].decode("utf-8").splitlines() try: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Downloading fresh PSL from {PSL_URL}") + LOGGER.debug("Cached PSL is missing or older than 1 day") + resp = get(PSL_URL, timeout=5) resp.raise_for_status() content = resp.text + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Downloaded PSL successfully, {len(content)} bytes") + LOGGER.debug(f"PSL contains {len(content.splitlines())} lines") + cached, err = JOB.cache_file(PSL_STATIC_FILE, content.encode("utf-8")) if not cached: LOGGER.error(f"Error while saving public suffix list to cache: {err}") + else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("PSL successfully cached for future use") + return content.splitlines() except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while downloading public suffix list: {e}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Download failed, checking for existing static file") + if PSL_STATIC_FILE.exists(): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using existing static PSL file: {PSL_STATIC_FILE}") with PSL_STATIC_FILE.open("r", encoding="utf-8") as f: return f.read().splitlines() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("No PSL data available - returning empty list") return [] -# Parse PSL lines into rules and exceptions sets. -# Processes the public suffix list format, handling comments, -# exceptions (lines starting with !), and regular rules. -# Args: psl_lines - List of lines from the PSL file -# Returns: dict - Contains 'rules' and 'exceptions' sets def parse_psl(psl_lines): + # Parse PSL lines into rules and exceptions sets. + # Processes the public suffix list format, handling comments, + # exceptions (lines starting with !), and regular rules. + # + # Args: + # psl_lines - List of lines from the PSL file + # + # Returns: + # dict - Contains 'rules' and 'exceptions' sets if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Parsing {len(psl_lines)} PSL lines") + LOGGER.debug("Processing rules, exceptions, and filtering comments") rules = set() exceptions = set() + comments_skipped = 0 + empty_lines_skipped = 0 + for line in psl_lines: line = line.strip() - if not line or line.startswith("//"): + if not line: + empty_lines_skipped += 1 + continue + if line.startswith("//"): + comments_skipped += 1 continue if line.startswith("!"): exceptions.add(line[1:]) @@ -142,74 +180,122 @@ def parse_psl(psl_lines): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Parsed {len(rules)} rules and {len(exceptions)} exceptions") + LOGGER.debug(f"Skipped {comments_skipped} comments and {empty_lines_skipped} empty lines") + LOGGER.debug("PSL parsing completed successfully") return {"rules": rules, "exceptions": exceptions} -# Check if domain is forbidden by PSL rules. -# Validates whether a domain would be blacklisted according to the -# Public Suffix List rules and exceptions. -# Args: domain - Domain name to check -# psl - Parsed PSL data (dict with 'rules' and 'exceptions') -# Returns: bool - True if domain is blacklisted def is_domain_blacklisted(domain, psl): + # Check if domain is forbidden by PSL rules. + # Validates whether a domain would be blacklisted according to the + # Public Suffix List rules and exceptions. + # + # Args: + # domain - Domain name to check + # psl - Parsed PSL data (dict with 'rules' and 'exceptions') + # + # Returns: + # bool - True if domain is blacklisted domain = domain.lower().strip(".") labels = domain.split(".") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking domain {domain} against PSL rules") + LOGGER.debug(f"Domain has {len(labels)} labels: {labels}") + LOGGER.debug(f"PSL contains {len(psl['rules'])} rules and {len(psl['exceptions'])} exceptions") for i in range(len(labels)): candidate = ".".join(labels[i:]) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking candidate: {candidate}") + if candidate in psl["exceptions"]: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Domain {domain} allowed by PSL exception {candidate}") return False + if candidate in psl["rules"]: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Found PSL rule match: {candidate}") + LOGGER.debug(f"Checking blacklist conditions for i={i}") + if i == 0: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} blacklisted - exact PSL rule match") return True if i == 0 and domain.startswith("*."): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Wildcard domain {domain} blacklisted - exact PSL rule match") return True if i == 0 or (i == 1 and labels[0] == "*"): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} blacklisted - PSL rule violation") return True if len(labels[i:]) == len(labels): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} blacklisted - full label match") return True - if f"*.{candidate}" in psl["rules"]: + + wildcard_candidate = f"*.{candidate}" + if wildcard_candidate in psl["rules"]: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Found PSL wildcard rule match: {wildcard_candidate}") + if len(labels[i:]) == 2: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} blacklisted - wildcard PSL rule match") return True + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Domain {domain} not blacklisted by PSL") return False -# Get ACME server configuration for the specified CA provider. -# Returns the appropriate ACME server URL and name for the given -# certificate authority and environment (staging/production). -# Args: ca_provider - Certificate authority name ('zerossl' or 'letsencrypt') -# staging - Whether to use staging environment -# Returns: dict - Server URL and CA name def get_certificate_authority_config(ca_provider, staging=False): + # Get ACME server configuration for the specified CA provider. + # Returns the appropriate ACME server URL and name for the given + # certificate authority and environment (staging/production). + # + # Args: + # ca_provider - Certificate authority name ('zerossl' or 'letsencrypt') + # staging - Whether to use staging environment + # + # Returns: + # dict - Server URL and CA name if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Getting CA config for {ca_provider}, staging={staging}") if ca_provider.lower() == "zerossl": - return { + config = { "server": ZEROSSL_STAGING_SERVER if staging else ZEROSSL_ACME_SERVER, "name": "ZeroSSL" } else: # Default to Let's Encrypt - return { + config = { "server": (LETSENCRYPT_STAGING_SERVER if staging else LETSENCRYPT_ACME_SERVER), "name": "Let's Encrypt" } + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CA config: {config}") + + return config -# Setup External Account Binding (EAB) credentials for ZeroSSL. -# Contacts the ZeroSSL API to obtain EAB credentials required for -# ACME certificate issuance with ZeroSSL. -# Args: email - Email address for the account -# api_key - ZeroSSL API key -# Returns: tuple - (eab_kid, eab_hmac_key) or (None, None) on failure def setup_zerossl_eab_credentials(email, api_key=None): + # Setup External Account Binding (EAB) credentials for ZeroSSL. + # Contacts the ZeroSSL API to obtain EAB credentials required for + # ACME certificate issuance with ZeroSSL. + # + # Args: + # email - Email address for the account + # api_key - ZeroSSL API key + # + # Returns: + # tuple - (eab_kid, eab_hmac_key) or (None, None) on failure LOGGER.info(f"Setting up ZeroSSL EAB credentials for email: {email}") if not api_key: @@ -221,12 +307,17 @@ def setup_zerossl_eab_credentials(email, api_key=None): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Making request to ZeroSSL API for EAB credentials") + LOGGER.debug(f"Email: {email}") + LOGGER.debug(f"API key provided: {bool(api_key)}") LOGGER.info("Making request to ZeroSSL API for EAB credentials") # Try the correct ZeroSSL API endpoint try: # The correct endpoint for ZeroSSL EAB credentials + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Attempting primary ZeroSSL EAB endpoint") + response = get( "https://api.zerossl.com/acme/eab-credentials", headers={"Authorization": f"Bearer {api_key}"}, @@ -235,6 +326,7 @@ def setup_zerossl_eab_credentials(email, api_key=None): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"ZeroSSL API response status: {response.status_code}") + LOGGER.debug(f"Response headers: {dict(response.headers)}") LOGGER.info(f"ZeroSSL API response status: {response.status_code}") if response.status_code == 200: @@ -271,6 +363,9 @@ def setup_zerossl_eab_credentials(email, api_key=None): LOGGER.info(f"Primary endpoint response: {response_text}") # Try alternative endpoint with email parameter + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Attempting alternative ZeroSSL EAB endpoint") + response = get( "https://api.zerossl.com/acme/eab-credentials-email", params={"email": email}, @@ -312,6 +407,9 @@ def setup_zerossl_eab_credentials(email, api_key=None): LOGGER.debug(format_exc()) LOGGER.error(f"❌ Error setting up ZeroSSL EAB credentials: {e}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("ZeroSSL EAB setup failed with exception") + # Additional troubleshooting info LOGGER.error("Troubleshooting steps:") LOGGER.error("1. Verify your ZeroSSL API key is valid") @@ -322,12 +420,17 @@ def setup_zerossl_eab_credentials(email, api_key=None): return None, None -# Get CAA records for a domain using dig command. -# Queries DNS CAA records to check certificate authority authorization. -# Returns None if dig command is not available. -# Args: domain - Domain name to query -# Returns: list or None - List of CAA record dicts or None if unavailable def get_caa_records(domain): + # Get CAA records for a domain using dig command. + # Queries DNS CAA records to check certificate authority authorization. + # Returns None if dig command is not available. + # + # Args: + # domain - Domain name to query + # + # Returns: + # list or None - List of CAA record dicts or None if unavailable + # Check if dig command is available if not which("dig"): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -339,7 +442,9 @@ def get_caa_records(domain): # Use dig to query CAA records if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Querying CAA records for domain: {domain}") + LOGGER.debug("Using dig command with +short flag") LOGGER.info(f"Querying CAA records for domain: {domain}") + result = run( ["dig", "+short", domain, "CAA"], capture_output=True, @@ -347,12 +452,25 @@ def get_caa_records(domain): timeout=10 ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"dig command return code: {result.returncode}") + LOGGER.debug(f"dig stdout: {result.stdout}") + LOGGER.debug(f"dig stderr: {result.stderr}") + if result.returncode == 0 and result.stdout.strip(): LOGGER.info(f"Found CAA records for domain {domain}") caa_records = [] - for line in result.stdout.strip().split('\n'): + raw_lines = result.stdout.strip().split('\n') + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing {len(raw_lines)} CAA record lines") + + for line in raw_lines: line = line.strip() if line: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Parsing CAA record line: {line}") + # CAA record format: flags tag "value" # Example: 0 issue "letsencrypt.org" parts = line.split(' ', 2) @@ -365,9 +483,12 @@ def get_caa_records(domain): 'tag': tag, 'value': value }) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Parsed CAA record: flags={flags}, tag={tag}, value={value}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsed {len(caa_records)} CAA records for domain {domain}") + LOGGER.debug(f"Successfully parsed {len(caa_records)} CAA records for domain {domain}") LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain {domain}") return caa_records else: @@ -389,14 +510,18 @@ def get_caa_records(domain): return None -# Check if the CA provider is authorized by CAA records. -# Validates whether the certificate authority is permitted to issue -# certificates for the domain according to CAA DNS records. -# Args: domain - Domain name to check -# ca_provider - Certificate authority provider name -# is_wildcard - Whether this is for a wildcard certificate -# Returns: bool - True if CA is authorized or no CAA restrictions exist def check_caa_authorization(domain, ca_provider, is_wildcard=False): + # Check if the CA provider is authorized by CAA records. + # Validates whether the certificate authority is permitted to issue + # certificates for the domain according to CAA DNS records. + # + # Args: + # domain - Domain name to check + # ca_provider - Certificate authority provider name + # is_wildcard - Whether this is for a wildcard certificate + # + # Returns: + # bool - True if CA is authorized or no CAA restrictions exist if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( f"Checking CAA authorization for domain: {domain}, " @@ -417,14 +542,20 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): allowed_identifiers = ca_identifiers.get(ca_provider.lower(), []) if not allowed_identifiers: LOGGER.warning(f"Unknown CA provider for CAA check: {ca_provider}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Returning True for unknown CA provider (conservative approach)") return True # Allow unknown providers (conservative approach) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CA identifiers for {ca_provider}: {allowed_identifiers}") + # Check CAA records for the domain and parent domains check_domain = domain.lstrip("*.") domain_parts = check_domain.split(".") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Will check CAA records for domain chain: {check_domain}") + LOGGER.debug(f"Domain parts: {domain_parts}") LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") for i in range(len(domain_parts)): @@ -438,6 +569,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): if caa_records is None: # dig not available, skip CAA check LOGGER.info("CAA record checking skipped (dig command not available)") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Returning True due to unavailable dig command") return True if caa_records: @@ -475,6 +608,7 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Using CAA {record_type} records for authorization check") + LOGGER.debug(f"Records to check: {check_records}") LOGGER.info(f"Using CAA {record_type} records for authorization check") if not check_records: @@ -501,11 +635,16 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): ) for identifier in allowed_identifiers: for record in check_records: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Comparing identifier '{identifier}' with record '{record}'") + # Handle explicit deny (empty value or semicolon) if record == ";" or record.strip() == "": LOGGER.warning( f"CAA {record_type} record explicitly denies all CAs" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Found explicit deny record - authorization failed") return False # Check for CA authorization @@ -515,6 +654,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"✓ CA {ca_provider} ({identifier}) authorized by " f"CAA {record_type} record" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Authorization found: {identifier} in {record}") break if authorized: break @@ -531,34 +672,45 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): LOGGER.error( f"But {ca_provider} uses: {', '.join(allowed_identifiers)}" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("CAA authorization failed - no matching identifiers") return False # If we found CAA records and we're authorized, we can stop checking parent domains LOGGER.info(f"✓ CAA authorization successful for {domain}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("CAA authorization successful - stopping parent domain checks") return True # No CAA records found in the entire chain LOGGER.info( f"No CAA records found for {check_domain} or parent domains - any CA allowed" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("No CAA records found in entire domain chain - allowing any CA") return True -# Validate that all domains have valid A/AAAA records and CAA authorization -# for HTTP challenge. -# Checks DNS resolution and certificate authority authorization for each -# domain in the list to ensure HTTP challenge will succeed. -# Args: domains_list - List of domain names to validate -# ca_provider - Certificate authority provider name -# is_wildcard - Whether this is for wildcard certificates -# Returns: bool - True if all domains are valid for HTTP challenge def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", is_wildcard=False): + # Validate that all domains have valid A/AAAA records and CAA authorization + # for HTTP challenge. + # Checks DNS resolution and certificate authority authorization for each + # domain in the list to ensure HTTP challenge will succeed. + # + # Args: + # domains_list - List of domain names to validate + # ca_provider - Certificate authority provider name + # is_wildcard - Whether this is for wildcard certificates + # + # Returns: + # bool - True if all domains are valid for HTTP challenge if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( f"Validating {len(domains_list)} domains for HTTP challenge: " f"{', '.join(domains_list)}" ) + LOGGER.debug(f"CA provider: {ca_provider}, wildcard: {is_wildcard}") LOGGER.info( f"Validating {len(domains_list)} domains for HTTP challenge: " f"{', '.join(domains_list)}" @@ -569,6 +721,9 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", # Check if CAA validation should be skipped skip_caa_check = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CAA check will be {'skipped' if skip_caa_check else 'performed'}") + # Get external IPs once for all domain checks external_ips = get_external_ip() if external_ips: @@ -581,20 +736,39 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", "Could not determine server external IP - skipping IP match validation" ) + validation_passed = 0 + validation_failed = 0 + for domain in domains_list: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Validating domain: {domain}") + # Check DNS A/AAAA records with retry mechanism if not check_domain_a_record(domain, external_ips): invalid_domains.append(domain) + validation_failed += 1 + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"DNS validation failed for {domain}") continue # Check CAA authorization if not skip_caa_check: if not check_caa_authorization(domain, ca_provider, is_wildcard): caa_blocked_domains.append(domain) + validation_failed += 1 + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CAA authorization failed for {domain}") else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") LOGGER.info(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") + + validation_passed += 1 + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Validation passed for {domain}") + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Validation summary: {validation_passed} passed, {validation_failed} failed") # Report results if invalid_domains: @@ -626,14 +800,17 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", return True -# Get the external/public IP addresses of this server (both IPv4 and IPv6). -# Queries multiple external services to determine the server's public -# IP addresses for DNS validation purposes. -# Returns: dict or None - Dict with 'ipv4' and 'ipv6' keys, or None if all fail def get_external_ip(): + # Get the external/public IP addresses of this server (both IPv4 and IPv6). + # Queries multiple external services to determine the server's public + # IP addresses for DNS validation purposes. + # + # Returns: + # dict or None - Dict with 'ipv4' and 'ipv6' keys, or None if all fail if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Getting external IP addresses for server") LOGGER.info("Getting external IP addresses for server") + ipv4_services = [ "https://ipv4.icanhazip.com", "https://api.ipify.org", @@ -652,9 +829,14 @@ def get_external_ip(): # Try to get IPv4 address if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Attempting to get external IPv4 address") + LOGGER.debug(f"Trying {len(ipv4_services)} IPv4 services") LOGGER.info("Attempting to get external IPv4 address") - for service in ipv4_services: + + for i, service in enumerate(ipv4_services): try: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Trying IPv4 service {i+1}/{len(ipv4_services)}: {service}") + if "jsonip.com" in service: # This service returns JSON format response = get(service, timeout=5) @@ -667,6 +849,9 @@ def get_external_ip(): response.raise_for_status() ip = response.text.strip() + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Service returned: {ip}") + # Basic IPv4 validation if ip and "." in ip and len(ip.split(".")) == 4: try: @@ -679,6 +864,8 @@ def get_external_ip(): LOGGER.info(f"Successfully obtained external IPv4 address: {ip}") break except gaierror: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Invalid IPv4 address returned: {ip}") continue except BaseException as e: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -689,9 +876,14 @@ def get_external_ip(): # Try to get IPv6 address if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Attempting to get external IPv6 address") + LOGGER.debug(f"Trying {len(ipv6_services)} IPv6 services") LOGGER.info("Attempting to get external IPv6 address") - for service in ipv6_services: + + for i, service in enumerate(ipv6_services): try: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Trying IPv6 service {i+1}/{len(ipv6_services)}: {service}") + if "jsonip.com" in service: response = get(service, timeout=5) response.raise_for_status() @@ -702,6 +894,9 @@ def get_external_ip(): response.raise_for_status() ip = response.text.strip() + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Service returned: {ip}") + # Basic IPv6 validation if ip and ":" in ip: try: @@ -714,6 +909,8 @@ def get_external_ip(): LOGGER.info(f"Successfully obtained external IPv6 address: {ip}") break except gaierror: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Invalid IPv6 address returned: {ip}") continue except BaseException as e: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -725,6 +922,8 @@ def get_external_ip(): LOGGER.warning( "Could not determine external IP address (IPv4 or IPv6) from any service" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("All external IP services failed") return None LOGGER.info( @@ -735,13 +934,17 @@ def get_external_ip(): return external_ips -# Check if domain has valid A/AAAA records for HTTP challenge. -# Validates DNS resolution and optionally checks if the domain's -# IP addresses match the server's external IPs. -# Args: domain - Domain name to check -# external_ips - Dict with server's external IPv4/IPv6 addresses -# Returns: bool - True if domain has valid DNS records def check_domain_a_record(domain, external_ips=None): + # Check if domain has valid A/AAAA records for HTTP challenge. + # Validates DNS resolution and optionally checks if the domain's + # IP addresses match the server's external IPs. + # + # Args: + # domain - Domain name to check + # external_ips - Dict with server's external IPv4/IPv6 addresses + # + # Returns: + # bool - True if domain has valid DNS records if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking DNS A/AAAA records for domain: {domain}") LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") @@ -749,14 +952,24 @@ def check_domain_a_record(domain, external_ips=None): # Remove wildcard prefix if present check_domain = domain.lstrip("*.") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking domain after wildcard removal: {check_domain}") + # Attempt to resolve the domain to IP addresses result = getaddrinfo(check_domain, None) if result: ipv4_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET] ipv6_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET6] + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"DNS resolution results:") + LOGGER.debug(f" IPv4 addresses: {ipv4_addresses}") + LOGGER.debug(f" IPv6 addresses: {ipv6_addresses}") + if not ipv4_addresses and not ipv6_addresses: LOGGER.warning(f"Domain {check_domain} has no A or AAAA records") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("No valid IP addresses found in DNS resolution") return False # Log found addresses @@ -786,6 +999,9 @@ def check_domain_a_record(domain, external_ips=None): ipv4_match = False ipv6_match = False + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Checking IP address matches with server external IPs") + # Check IPv4 match if external_ips.get("ipv4") and ipv4_addresses: if external_ips["ipv4"] in ipv4_addresses: @@ -826,6 +1042,10 @@ def check_domain_a_record(domain, external_ips=None): has_any_match = ipv4_match or ipv6_match has_external_ip = external_ips.get("ipv4") or external_ips.get("ipv6") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"IP match results: IPv4={ipv4_match}, IPv6={ipv6_match}") + LOGGER.debug(f"Has external IP: {has_external_ip}, Has match: {has_any_match}") + if has_external_ip and not has_any_match: LOGGER.warning( f"⚠ Domain {check_domain} records do not match " @@ -842,9 +1062,13 @@ def check_domain_a_record(domain, external_ips=None): f"Strict IP check enabled - rejecting certificate " f"request for {check_domain}" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Strict IP check failed - returning False") return False LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("DNS validation completed successfully") return True else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -864,27 +1088,11 @@ def check_domain_a_record(domain, external_ips=None): except BaseException as e: LOGGER.info(format_exc()) LOGGER.error(f"Error checking DNS records for domain {check_domain}: {e}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("DNS check failed with unexpected exception") return False -# Execute certbot with retry mechanism. -# Wrapper around certbot_new that implements automatic retries with -# exponential backoff for failed certificate generation attempts. -# Args: challenge_type - Type of ACME challenge ('dns' or 'http') -# domains - Comma-separated list of domain names -# email - Email address for certificate registration -# provider - DNS provider name (for DNS challenge) -# credentials_path - Path to credentials file -# propagation - DNS propagation time in seconds -# profile - Certificate profile to use -# staging - Whether to use staging environment -# force - Force renewal of existing certificates -# cmd_env - Environment variables for certbot process -# max_retries - Maximum number of retry attempts -# ca_provider - Certificate authority provider -# api_key - API key for CA (if required) -# server_name - Server name for multisite configurations -# Returns: int - Exit code (0 for success) def certbot_new_with_retry( challenge_type: Literal["dns", "http"], domains: str, @@ -901,6 +1109,33 @@ def certbot_new_with_retry( api_key: str = None, server_name: str = None, ) -> int: + # Execute certbot with retry mechanism. + # Wrapper around certbot_new that implements automatic retries with + # exponential backoff for failed certificate generation attempts. + # + # Args: + # challenge_type - Type of ACME challenge ('dns' or 'http') + # domains - Comma-separated list of domain names + # email - Email address for certificate registration + # provider - DNS provider name (for DNS challenge) + # credentials_path - Path to credentials file + # propagation - DNS propagation time in seconds + # profile - Certificate profile to use + # staging - Whether to use staging environment + # force - Force renewal of existing certificates + # cmd_env - Environment variables for certbot process + # max_retries - Maximum number of retry attempts + # ca_provider - Certificate authority provider + # api_key - API key for CA (if required) + # server_name - Server name for multisite configurations + # + # Returns: + # int - Exit code (0 for success) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Starting certbot with retry for domains: {domains}") + LOGGER.debug(f"Max retries: {max_retries}, CA: {ca_provider}") + LOGGER.debug(f"Challenge: {challenge_type}, Provider: {provider}") + attempt = 1 while attempt <= max_retries + 1: if attempt > 1: @@ -912,9 +1147,13 @@ def certbot_new_with_retry( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Waiting {wait_time} seconds before retry...") + LOGGER.debug(f"Exponential backoff: base=30s, attempt={attempt}") LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Executing certbot attempt {attempt}") + result = certbot_new( challenge_type, domains, @@ -934,34 +1173,23 @@ def certbot_new_with_retry( if result == 0: if attempt > 1: LOGGER.info(f"Certificate generation succeeded on attempt {attempt}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Certbot completed successfully") return result if attempt >= max_retries + 1: LOGGER.error(f"Certificate generation failed after {max_retries + 1} attempts") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Maximum retries reached - giving up") return result + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Attempt {attempt} failed, will retry") attempt += 1 return result -# Generate new certificate using certbot. -# Main function to request SSL/TLS certificates from a certificate authority -# using the ACME protocol via certbot. -# Args: challenge_type - Type of ACME challenge ('dns' or 'http') -# domains - Comma-separated list of domain names -# email - Email address for certificate registration -# provider - DNS provider name (for DNS challenge) -# credentials_path - Path to credentials file -# propagation - DNS propagation time in seconds -# profile - Certificate profile to use -# staging - Whether to use staging environment -# force - Force renewal of existing certificates -# cmd_env - Environment variables for certbot process -# ca_provider - Certificate authority provider -# api_key - API key for CA (if required) -# server_name - Server name for multisite configurations -# Returns: int - Exit code (0 for success) def certbot_new( challenge_type: Literal["dns", "http"], domains: str, @@ -977,6 +1205,27 @@ def certbot_new( api_key: str = None, server_name: str = None, ) -> int: + # Generate new certificate using certbot. + # Main function to request SSL/TLS certificates from a certificate authority + # using the ACME protocol via certbot. + # + # Args: + # challenge_type - Type of ACME challenge ('dns' or 'http') + # domains - Comma-separated list of domain names + # email - Email address for certificate registration + # provider - DNS provider name (for DNS challenge) + # credentials_path - Path to credentials file + # propagation - DNS propagation time in seconds + # profile - Certificate profile to use + # staging - Whether to use staging environment + # force - Force renewal of existing certificates + # cmd_env - Environment variables for certbot process + # ca_provider - Certificate authority provider + # api_key - API key for CA (if required) + # server_name - Server name for multisite configurations + # + # Returns: + # int - Exit code (0 for success) if isinstance(credentials_path, str): credentials_path = Path(credentials_path) @@ -984,6 +1233,9 @@ def certbot_new( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Building certbot command for {domains}") + LOGGER.debug(f"CA config: {ca_config}") + LOGGER.debug(f"Challenge type: {challenge_type}") + LOGGER.debug(f"Profile: {profile}") command = [ CERTBOT_BIN, @@ -1046,6 +1298,11 @@ def certbot_new( eab_hmac_env = (getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "")) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Manual EAB credentials check:") + LOGGER.debug(f" EAB KID provided: {bool(eab_kid_env)}") + LOGGER.debug(f" EAB HMAC provided: {bool(eab_hmac_env)}") + if eab_kid_env and eab_hmac_env: LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials from environment") command.extend(["--eab-kid", eab_kid_env, "--eab-hmac-key", eab_hmac_env]) @@ -1078,6 +1335,12 @@ def certbot_new( if challenge_type == "dns": command.append("--preferred-challenges=dns") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"DNS challenge configuration:") + LOGGER.debug(f" Provider: {provider}") + LOGGER.debug(f" Propagation: {propagation}") + LOGGER.debug(f" Credentials path: {credentials_path}") + if propagation != "default": if not propagation.isdigit(): LOGGER.warning( @@ -1086,8 +1349,12 @@ def certbot_new( ) else: command.extend([f"--dns-{provider}-propagation-seconds", propagation]) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Set DNS propagation time to {propagation} seconds") if provider == "route53": + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Route53 provider - setting environment variables") with credentials_path.open("r") as file: for line in file: key, value = line.strip().split("=", 1) @@ -1097,10 +1364,17 @@ def certbot_new( if provider in ("desec", "infomaniak", "ionos", "njalla", "scaleway"): command.extend(["--authenticator", f"dns-{provider}"]) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using explicit authenticator for {provider}") else: command.append(f"--dns-{provider}") elif challenge_type == "http": + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("HTTP challenge configuration:") + LOGGER.debug(f" Auth hook: {JOBS_PATH.joinpath('certbot-auth.py')}") + LOGGER.debug(f" Cleanup hook: {JOBS_PATH.joinpath('certbot-cleanup.py')}") + command.extend( [ "--manual", @@ -1114,9 +1388,13 @@ def certbot_new( if force: command.append("--force-renewal") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Force renewal enabled") if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": command.append("-v") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Verbose mode enabled for certbot") LOGGER.info(f"Executing certbot command for {domains}") # Show command but mask sensitive EAB values for security @@ -1134,18 +1412,26 @@ def certbot_new( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Command: {' '.join(safe_command)}") + LOGGER.debug(f"Environment variables: {len(cmd_env)} items") + for key in cmd_env.keys(): + LOGGER.debug(f" {key}: {'***MASKED***' if 'key' in key.lower() or 'secret' in key.lower() or 'token' in key.lower() else 'set'}") LOGGER.info(f"Command: {' '.join(safe_command)}") current_date = datetime.now() + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Starting certbot process") + process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, env=cmd_env) + lines_processed = 0 while process.poll() is None: if process.stderr: rlist, _, _ = select([process.stderr], [], [], 2) if rlist: for line in process.stderr: LOGGER_CERTBOT.info(line.strip()) + lines_processed += 1 break if datetime.now() - current_date > timedelta(seconds=5): @@ -1158,18 +1444,34 @@ def certbot_new( f"{challenge_info}..." ) current_date = datetime.now() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certbot still running, processed {lines_processed} output lines") + + final_return_code = process.returncode + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certbot process completed with return code: {final_return_code}") + LOGGER.debug(f"Total output lines processed: {lines_processed}") - return process.returncode + return final_return_code +# Global configuration and setup IS_MULTISITE = getenv("MULTISITE", "no") == "yes" try: + # Main execution block for certificate generation servers = getenv("SERVER_NAME", "www.example.com").lower() or [] if isinstance(servers, str): servers = servers.split(" ") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Server configuration detected:") + LOGGER.debug(f" Multisite mode: {IS_MULTISITE}") + LOGGER.debug(f" Server count: {len(servers)}") + LOGGER.debug(f" Servers: {servers}") + if not servers: LOGGER.warning("There are no server names, skipping generation...") sys_exit(0) @@ -1181,6 +1483,11 @@ def certbot_new( use_letsencrypt = getenv("AUTO_LETS_ENCRYPT", "no") == "yes" use_letsencrypt_dns = getenv("LETS_ENCRYPT_CHALLENGE", "http") == "dns" domains_server_names = {servers[0]: " ".join(servers).lower()} + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Single-site configuration:") + LOGGER.debug(f" Let's Encrypt enabled: {use_letsencrypt}") + LOGGER.debug(f" DNS challenge: {use_letsencrypt_dns}") else: domains_server_names = {} @@ -1195,6 +1502,12 @@ def certbot_new( domains_server_names[first_server] = getenv( f"{first_server}_SERVER_NAME", first_server ).lower() + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Multi-site configuration:") + LOGGER.debug(f" Let's Encrypt enabled anywhere: {use_letsencrypt}") + LOGGER.debug(f" DNS challenge used anywhere: {use_letsencrypt_dns}") + LOGGER.debug(f" Domain mappings: {domains_server_names}") if not use_letsencrypt: LOGGER.info("Let's Encrypt is not activated, skipping generation...") @@ -1203,6 +1516,9 @@ def certbot_new( provider_classes = {} if use_letsencrypt_dns: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("DNS challenge detected - loading provider classes") + provider_classes: Dict[ str, Union[ @@ -1249,6 +1565,8 @@ def certbot_new( JOB = Job(LOGGER, __file__) # Restore data from db cache of certbot-renew job + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Restoring certificate data from database cache") JOB.restore_cache(job_name="certbot-renew") env = { @@ -1265,6 +1583,9 @@ def certbot_new( if getenv("DATABASE_URI"): env["DATABASE_URI"] = getenv("DATABASE_URI") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Checking existing certificates") + proc = run( [ CERTBOT_BIN, @@ -1293,8 +1614,17 @@ def certbot_new( if proc.returncode != 0: LOGGER.error(f"Error while checking certificates:\n{proc.stdout}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Certificate listing failed - proceeding anyway") else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Certificate listing successful - analyzing existing certificates") + certificate_blocks = stdout.split("Certificate Name: ")[1:] + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Found {len(certificate_blocks)} existing certificates") + for first_server, domains in domains_server_names.items(): if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): @@ -1306,12 +1636,20 @@ def certbot_new( ) original_first_server = deepcopy(first_server) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing server: {first_server}") + LOGGER.debug(f" Challenge: {letsencrypt_challenge}") + LOGGER.debug(f" Domains: {domains}") + if ( letsencrypt_challenge == "dns" and (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes" ): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using wildcard mode for {first_server}") + wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( (first_server,) ) @@ -1368,6 +1706,11 @@ def certbot_new( else set(domains.split()) ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate domain comparison for {first_server}:") + LOGGER.debug(f" Existing: {sorted(cert_domains_set)}") + LOGGER.debug(f" Desired: {sorted(desired_domains_set)}") + if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 LOGGER.warning( @@ -1400,6 +1743,11 @@ def certbot_new( else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes" ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"CA server comparison for {first_server}:") + LOGGER.debug(f" Current: {current_server}") + LOGGER.debug(f" Expected: {expected_config['server']}") + if current_server and current_server != expected_config["server"]: domains_to_ask[first_server] = 2 LOGGER.warning( @@ -1415,6 +1763,11 @@ def certbot_new( ) == "yes" is_test_cert = "TEST_CERT" in cert_domains.group("expiry_date") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Staging environment check for {first_server}:") + LOGGER.debug(f" Use staging: {use_staging}") + LOGGER.debug(f" Is test cert: {is_test_cert}") + if (is_test_cert and not use_staging) or (not is_test_cert and use_staging): domains_to_ask[first_server] = 2 LOGGER.warning( @@ -1446,6 +1799,11 @@ def certbot_new( current_provider = value.strip().replace("dns-", "") break + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Provider comparison for {first_server}:") + LOGGER.debug(f" Current: {current_provider}") + LOGGER.debug(f" Configured: {provider}") + if letsencrypt_challenge == "dns": if provider and current_provider != provider: domains_to_ask[first_server] = 2 @@ -1458,6 +1816,9 @@ def certbot_new( # Check if DNS credentials have changed if provider and current_provider == provider: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Checking DNS credentials for {first_server}") + credential_key = ( f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE @@ -1551,6 +1912,10 @@ def certbot_new( psl_lines = None psl_rules = None + certificates_generated = 0 + certificates_failed = 0 + + # Process each server configuration for first_server, domains in domains_server_names.items(): if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): @@ -1636,6 +2001,8 @@ def certbot_new( ).strip() if custom_profile: data["profile"] = custom_profile + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Using custom profile: {custom_profile}") if data["challenge"] == "http" and data["use_wildcard"]: LOGGER.warning( @@ -1650,6 +2017,8 @@ def certbot_new( .lstrip("*.") ) ): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"No certificate needed for {first_server}") continue if not data["max_retries"].isdigit(): @@ -1663,6 +2032,9 @@ def certbot_new( # Getting the DNS provider data if necessary if data["challenge"] == "dns": + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing DNS credentials for {first_server}") + credential_key = ( f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE @@ -1732,7 +2104,11 @@ def certbot_new( data["credential_items"][key] = value if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Data for service {first_server}: {dumps(data)}") + safe_data = data.copy() + safe_data["credential_items"] = {k: "***MASKED***" for k in data["credential_items"].keys()} + if data["api_key"]: + safe_data["api_key"] = "***MASKED***" + LOGGER.debug(f"Safe data for service {first_server}: {dumps(safe_data)}") # Validate CA provider and API key requirements LOGGER.info( @@ -1813,8 +2189,12 @@ def certbot_new( if data["check_psl"]: if psl_lines is None: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Loading PSL for wildcard domain validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Parsing PSL rules") psl_rules = parse_psl(psl_lines) wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( @@ -1841,8 +2221,12 @@ def certbot_new( LOGGER.info(f"[{first_server}] Wildcard group {group}") elif data["check_psl"]: if psl_lines is None: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Loading PSL for regular domain validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Parsing PSL rules") psl_rules = parse_psl(psl_lines) for d in domains.split(): @@ -1855,12 +2239,17 @@ def certbot_new( break if is_blacklisted: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Skipping {first_server} due to PSL blacklist") continue # Generating the credentials file credentials_path = CACHE_PATH.joinpath(*file_path) if data["challenge"] == "dns": + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Managing credentials file for {first_server}: {credentials_path}") + if not credentials_path.is_file(): cached, err = JOB.cache_file( credentials_path.name, content, job_name="certbot-renew", @@ -1913,6 +2302,8 @@ def certbot_new( credentials_path.chmod(0o600) if data["use_wildcard"]: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Wildcard processing complete for {first_server}") continue domains = domains.replace(" ", ",") @@ -1920,9 +2311,12 @@ def certbot_new( LOGGER.info( f"Asking {ca_name} certificates for domain(s): {domains} " f"(email = {data['email']}){' using staging' if data['staging'] else ''} " - f"with {data['challenge']} challenge, using {data['profile']!r} profile..." + f" with {data['challenge']} challenge, using {data['profile']!r} profile..." ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Requesting certificate for {domains}") + if ( certbot_new_with_retry( data["challenge"], @@ -1943,9 +2337,11 @@ def certbot_new( != 0 ): status = 2 + certificates_failed += 1 LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") else: status = 1 if status == 0 else status + certificates_generated += 1 LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") generated_domains.update(domains.split(",")) @@ -1953,6 +2349,9 @@ def certbot_new( # Generating the wildcards if necessary wildcards = WILDCARD_GENERATOR.get_wildcards() if wildcards: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing {len(wildcards)} wildcard groups") + for group, data in wildcards.items(): if not data: continue @@ -1963,6 +2362,12 @@ def certbot_new( profile = group_parts[2] base_domain = group_parts[3] + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Processing wildcard group: {group}") + LOGGER.debug(f" Provider: {provider}") + LOGGER.debug(f" Profile: {profile}") + LOGGER.debug(f" Base domain: {base_domain}") + email = data.pop("email") credentials_file = CACHE_PATH.joinpath( f"{group}.{provider_classes[provider].get_file_type() if provider in provider_classes else 'txt'}" @@ -2011,6 +2416,9 @@ def certbot_new( base_domain = WILDCARD_GENERATOR.get_base_domain(domain) active_cert_names.add(base_domain) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Requesting wildcard certificate for {domains}") + if ( certbot_new_with_retry( "dns", @@ -2030,9 +2438,11 @@ def certbot_new( != 0 ): status = 2 + certificates_failed += 1 LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") else: status = 1 if status == 0 else status + certificates_generated += 1 LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") generated_domains.update(domains_split) @@ -2041,8 +2451,18 @@ def certbot_new( "No wildcard domains found, skipping wildcard certificate(s) generation..." ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate generation summary:") + LOGGER.debug(f" Generated: {certificates_generated}") + LOGGER.debug(f" Failed: {certificates_failed}") + LOGGER.debug(f" Total domains: {len(generated_domains)}") + if CACHE_PATH.is_dir(): # Clearing all missing credentials files + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Cleaning up old credentials files") + + cleaned_files = 0 for ext in ("*.ini", "*.env", "*.json"): for file in list(CACHE_PATH.rglob(ext)): if "etc" in file.parts or not file.is_file(): @@ -2055,6 +2475,10 @@ def certbot_new( service_id=(file.parent.name if file.parent.name != "letsencrypt" else "") ) + cleaned_files += 1 + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Cleaned up {cleaned_files} old credentials files") # Clearing all no longer needed certificates if getenv("LETS_ENCRYPT_CLEAR_OLD_CERTS", "no") == "yes": @@ -2063,6 +2487,9 @@ def certbot_new( "used certificates..." ) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Starting certificate cleanup process") + # Get list of all certificates proc = run( [ @@ -2085,6 +2512,12 @@ def certbot_new( if proc.returncode == 0: certificate_blocks = proc.stdout.split("Certificate Name: ")[1:] + certificates_removed = 0 + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Found {len(certificate_blocks)} certificates to evaluate") + LOGGER.debug(f"Active certificates: {sorted(active_cert_names)}") + for block in certificate_blocks: cert_name = block.split("\n", 1)[0].strip() @@ -2123,6 +2556,7 @@ def certbot_new( if delete_proc.returncode == 0: LOGGER.info(f"Successfully deleted certificate {cert_name}") + certificates_removed += 1 cert_dir = DATA_PATH.joinpath("live", cert_name) archive_dir = DATA_PATH.joinpath("archive", cert_name) renewal_file = DATA_PATH.joinpath("renewal", f"{cert_name}.conf") @@ -2152,22 +2586,40 @@ def certbot_new( f"Failed to delete certificate {cert_name}: " f"{delete_proc.stdout}" ) + + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate cleanup completed - removed {certificates_removed} certificates") else: LOGGER.error(f"Error listing certificates: {proc.stdout}") # Save data to db cache if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Saving certificate data to database cache") + cached, err = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") if not cached: LOGGER.error(f"Error while saving data to db cache: {err}") else: LOGGER.info("Successfully saved data to db cache") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Database cache update completed") + else: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("No certificate data to cache") except SystemExit as e: status = e.code + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Script exiting via SystemExit with code: {e.code}") except BaseException as e: status = 1 LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-new.py:\n{e}") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Script failed with unexpected exception") + +if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Certificate generation process completed with status: {status}") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index 766b04abd0..c0cc8171ce 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -34,33 +34,40 @@ # This checks both single-site and multi-site deployment modes if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Starting Let's Encrypt certificate renewal process") - LOGGER.debug("Checking if Let's Encrypt is enabled") + LOGGER.debug("Checking if Let's Encrypt is enabled in configuration") + LOGGER.debug("Will check both single-site and multi-site modes") use_letsencrypt = False multisite_mode = getenv("MULTISITE", "no") == "yes" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Multisite mode: {multisite_mode}") + LOGGER.debug(f"Multisite mode detected: {multisite_mode}") + LOGGER.debug("Determining which Let's Encrypt check method to use") # Single-site mode: Check global AUTO_LETS_ENCRYPT setting if not multisite_mode: use_letsencrypt = getenv("AUTO_LETS_ENCRYPT", "no") == "yes" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Single-site mode - AUTO_LETS_ENCRYPT: {use_letsencrypt}") + LOGGER.debug("Checking single-site mode configuration") + LOGGER.debug(f"Global AUTO_LETS_ENCRYPT setting: {use_letsencrypt}") + LOGGER.debug("Single setting controls all domains in this mode") # Multi-site mode: Check per-server AUTO_LETS_ENCRYPT settings else: server_names = getenv("SERVER_NAME", "www.example.com").split(" ") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Multi-site mode - checking {len(server_names)} servers") - LOGGER.debug(f"Server names: {server_names}") + LOGGER.debug("Checking multi-site mode configuration") + LOGGER.debug(f"Found {len(server_names)} configured servers") + LOGGER.debug(f"Server list: {server_names}") + LOGGER.debug("Checking each server for Let's Encrypt enablement") # Check if any server has Let's Encrypt enabled for i, first_server in enumerate(server_names): if first_server: - server_le_enabled = getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes" + server_le_enabled = getenv(f"{first_server}_AUTO_LETS_ENCRYPT", + "no") == "yes" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( @@ -70,21 +77,28 @@ if server_le_enabled: use_letsencrypt = True + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug(f"Found Let's Encrypt enabled on {first_server}") + LOGGER.debug("At least one server needs renewal - proceeding") break # Exit early if Let's Encrypt is not configured if not use_letsencrypt: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Let's Encrypt not enabled, exiting renewal process") + LOGGER.debug("Let's Encrypt not enabled on any servers") + LOGGER.debug("No certificates to renew - exiting early") + LOGGER.debug("Renewal process skipped entirely") LOGGER.info("Let's Encrypt is not activated, skipping renew...") sys_exit(0) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Let's Encrypt is enabled, proceeding with renewal") + LOGGER.debug("Let's Encrypt is enabled - proceeding with renewal") + LOGGER.debug("Will attempt to renew all existing certificates") # Initialize job handler for caching operations if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Initializing job handler") + LOGGER.debug("Initializing job handler for database operations") + LOGGER.debug("Job handler manages certificate data caching") JOB = Job(LOGGER, __file__) @@ -92,6 +106,7 @@ # These control paths, timeouts, and configuration testing behavior if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Setting up environment for certbot execution") + LOGGER.debug("Configuring paths and operational parameters") env = { "PATH": getenv("PATH", ""), @@ -112,9 +127,14 @@ env["DATABASE_URI"] = getenv("DATABASE_URI") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Environment configuration:") - LOGGER.debug(f" PATH: {env['PATH'][:100]}..." if len(env['PATH']) > 100 else f" PATH: {env['PATH']}") - LOGGER.debug(f" PYTHONPATH: {env['PYTHONPATH'][:100]}..." if len(env['PYTHONPATH']) > 100 else f" PYTHONPATH: {env['PYTHONPATH']}") + LOGGER.debug("Environment configuration for certbot:") + path_display = (env['PATH'][:100] + "..." if len(env['PATH']) > 100 + else env['PATH']) + pythonpath_display = (env['PYTHONPATH'][:100] + "..." + if len(env['PYTHONPATH']) > 100 + else env['PYTHONPATH']) + LOGGER.debug(f" PATH: {path_display}") + LOGGER.debug(f" PYTHONPATH: {pythonpath_display}") LOGGER.debug(f" RELOAD_MIN_TIMEOUT: {env['RELOAD_MIN_TIMEOUT']}") LOGGER.debug(f" DISABLE_CONFIGURATION_TESTING: {env['DISABLE_CONFIGURATION_TESTING']}") LOGGER.debug(f" DATABASE_URI configured: {'Yes' if getenv('DATABASE_URI') else 'No'}") @@ -125,26 +145,29 @@ command = [ CERTBOT_BIN, "renew", - "--no-random-sleep-on-renew", + "--no-random-sleep-on-renew", # Disable random sleep for scheduled runs "--config-dir", - DATA_PATH.as_posix(), + DATA_PATH.as_posix(), # Where certificates are stored "--work-dir", - WORK_DIR, + WORK_DIR, # Temporary working directory "--logs-dir", - LOGS_DIR, + LOGS_DIR, # Log output directory ] # Add verbose flag if debug logging is enabled if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": command.append("-v") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Debug mode enabled, adding verbose flag to certbot") + LOGGER.debug("Debug mode enabled - adding verbose flag to certbot") + LOGGER.debug("Certbot will provide detailed output") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot command: {' '.join(command)}") - LOGGER.debug(f"Working directory: {WORK_DIR}") - LOGGER.debug(f"Config directory: {DATA_PATH.as_posix()}") - LOGGER.debug(f"Logs directory: {LOGS_DIR}") + LOGGER.debug("Certbot command configuration:") + LOGGER.debug(f" Command: {' '.join(command)}") + LOGGER.debug(f" Working directory: {WORK_DIR}") + LOGGER.debug(f" Config directory: {DATA_PATH.as_posix()}") + LOGGER.debug(f" Logs directory: {LOGS_DIR}") + LOGGER.debug("Command will check all existing certificates for renewal") LOGGER.info("Starting certificate renewal process") @@ -152,59 +175,77 @@ # Process output is captured and logged through our logger if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Executing certbot renew command") + LOGGER.debug("Will capture and relay all certbot output") + LOGGER.debug("Process runs with isolated environment") process = Popen( command, - stdin=DEVNULL, - stderr=PIPE, - universal_newlines=True, - env=env, + stdin=DEVNULL, # No input needed + stderr=PIPE, # Capture error output + universal_newlines=True, # Text mode + env=env, # Controlled environment ) # Stream certbot output to our logger in real-time # This ensures all certbot messages are captured in BunkerWeb logs line_count = 0 + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Starting real-time output capture from certbot") + while process.poll() is None: if process.stderr: for line in process.stderr: line_count += 1 LOGGER_CERTBOT.info(line.strip()) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG" and line_count % 10 == 0: - LOGGER.debug(f"Processed {line_count} certbot output lines") + if (getenv("LOG_LEVEL", "INFO").upper() == "DEBUG" + and line_count % 10 == 0): + LOGGER.debug(f"Processed {line_count} lines of certbot output") # Wait for process completion and check return code final_return_code = process.returncode if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot process completed with return code: {final_return_code}") - LOGGER.debug(f"Total certbot output lines processed: {line_count}") + LOGGER.debug("Certbot process completed") + LOGGER.debug(f"Final return code: {final_return_code}") + LOGGER.debug(f"Total output lines processed: {line_count}") + LOGGER.debug("Analyzing return code to determine success/failure") # Handle renewal results if final_return_code != 0: status = 2 LOGGER.error("Certificates renewal failed") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Renewal process failed, certificate data will not be cached") + LOGGER.debug("Certbot returned non-zero exit code") + LOGGER.debug("Certificate renewal process failed") + LOGGER.debug("Will not cache certificate data due to failure") else: LOGGER.info("Certificate renewal completed successfully") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Renewal process succeeded, proceeding to cache certificate data") + LOGGER.debug("Certbot completed successfully") + LOGGER.debug("All eligible certificates have been renewed") + LOGGER.debug("Proceeding to cache updated certificate data") # Save Let's Encrypt certificate data to database cache # This ensures certificate data is available for distribution to cluster nodes if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking if certificate data directory exists") - LOGGER.debug(f"Data path: {DATA_PATH}") + LOGGER.debug("Checking certificate data directory for caching") + LOGGER.debug(f"Certificate data path: {DATA_PATH}") LOGGER.debug(f"Directory exists: {DATA_PATH.is_dir()}") if DATA_PATH.is_dir(): dir_contents = list(DATA_PATH.iterdir()) - LOGGER.debug(f"Directory contents count: {len(dir_contents)}") + LOGGER.debug(f"Directory contains {len(dir_contents)} items") + LOGGER.debug("Directory listing:") + for item in dir_contents[:5]: # Show first 5 items + LOGGER.debug(f" {item.name}") + if len(dir_contents) > 5: + LOGGER.debug(f" ... and {len(dir_contents) - 5} more items") # Only cache if directory exists and contains files if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Caching certificate data to database") + LOGGER.debug("Certificate data found - proceeding with caching") + LOGGER.debug("This will store certificates in database for cluster distribution") cached, err = JOB.cache_dir(DATA_PATH) if not cached: @@ -213,29 +254,43 @@ ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Cache operation failed with error: {err}") + LOGGER.debug("Certificates renewed but not cached for distribution") else: LOGGER.info("Successfully saved Let's Encrypt data to db cache") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Certificate data successfully cached to database") + LOGGER.debug("Cached certificates available for cluster distribution") else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No certificate data to cache (directory empty or missing)") + LOGGER.debug("No certificate data directory found or directory empty") + LOGGER.debug("This may be normal if no certificates needed renewal") LOGGER.warning("No certificate data found to cache") except SystemExit as e: status = e.code if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Script exiting with SystemExit code: {e.code}") + LOGGER.debug(f"Script exiting via SystemExit with code: {e.code}") + LOGGER.debug("This is typically a normal exit condition") except BaseException as e: status = 2 LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-renew.py:\n{e}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + LOGGER.debug("Unexpected exception occurred during renewal") LOGGER.debug("Full exception traceback logged above") LOGGER.debug("Setting exit status to 2 due to unexpected exception") + LOGGER.debug("Renewal process aborted due to error") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate renewal process completed with status: {status}") + LOGGER.debug(f"Certificate renewal process completed with final status: {status}") + if status == 0: + LOGGER.debug("Renewal process completed successfully") + LOGGER.debug("All certificates are up to date") + elif status == 2: + LOGGER.debug("Renewal process failed") + LOGGER.debug("Manual intervention may be required") + else: + LOGGER.debug(f"Renewal completed with status {status}") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/letsencrypt.py b/src/common/core/letsencrypt/jobs/letsencrypt.py index 31a09a6e68..e50b0cd33e 100644 --- a/src/common/core/letsencrypt/jobs/letsencrypt.py +++ b/src/common/core/letsencrypt/jobs/letsencrypt.py @@ -16,27 +16,38 @@ sys_path.append(python_path_str) -# Factory function for creating a model_validator for alias mapping. -# This allows DNS providers to accept credentials under multiple field names -# for better compatibility with different configuration formats. -# Args: field_map - Dictionary mapping canonical field names to list of aliases -# Returns: Configured model_validator function def alias_model_validator(field_map: dict): + # Factory function for creating a model_validator for alias mapping. + # This allows DNS providers to accept credentials under multiple field + # names for better compatibility with different configuration formats. + # + # Args: + # field_map: Dictionary mapping canonical field names to list of + # aliases + # + # Returns: + # Configured model_validator function def validator(cls, values): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Processing aliases for {cls.__name__}") print(f"DEBUG: Input values: {list(values.keys())}") + print(f"DEBUG: Field mapping has {len(field_map)} canonical fields") for field, aliases in field_map.items(): + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Checking field '{field}' with {len(aliases)} aliases") + for alias in aliases: if alias in values: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Mapping alias '{alias}' to field '{field}'") + print(f"DEBUG: Found alias '{alias}' for field '{field}'") + print(f"DEBUG: Mapping alias '{alias}' to canonical field '{field}'") values[field] = values[alias] break if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Final mapped values: {list(values.keys())}") + print(f"DEBUG: Alias processing completed for {cls.__name__}") return values @@ -45,20 +56,23 @@ def validator(cls, values): class Provider(BaseModel): # Base class for DNS providers. - # Provides common functionality for credential formatting and file type handling. - # All DNS provider classes inherit from this base class. + # Provides common functionality for credential formatting and file type + # handling. All DNS provider classes inherit from this base class. model_config = ConfigDict(extra="ignore") - # Return the formatted credentials to be written to a file. - # Default implementation creates INI-style key=value format. - # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: + # Return the formatted credentials to be written to a file. + # Default implementation creates INI-style key=value format. + # + # Returns: + # bytes - UTF-8 encoded credential content if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": excluded_fields = {"file_type"} fields = self.model_dump(exclude=excluded_fields) print(f"DEBUG: {self.__class__.__name__} formatting {len(fields)} fields") print(f"DEBUG: Excluded fields: {excluded_fields}") + print(f"DEBUG: Using default INI-style key=value format") content = "\n".join( f"{key} = {value}" @@ -67,20 +81,24 @@ def get_formatted_credentials(self) -> bytes: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Generated {len(content)} bytes of credential content") + print(f"DEBUG: Content will be written as UTF-8 encoded text") return content - # Return the file type that the credentials should be written to. - # Default implementation returns 'ini' for most providers. - # Returns: Literal type indicating file extension @staticmethod def get_file_type() -> Literal["ini"]: + # Return the file type that the credentials should be written to. + # Default implementation returns 'ini' for most providers. + # + # Returns: + # Literal type indicating file extension return "ini" class CloudflareProvider(Provider): - # Supports both API token (recommended) and legacy email/API key authentication. - # Requires either api_token OR both email and api_key for authentication. + # Supports both API token (recommended) and legacy email/API key + # authentication. Requires either api_token OR both email and api_key + # for authentication. dns_cloudflare_api_token: str = "" dns_cloudflare_email: str = "" @@ -100,10 +118,13 @@ class CloudflareProvider(Provider): } ) - # Return the formatted credentials, excluding defaults. - # Only includes non-empty credential fields to avoid cluttering output. - # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: + # Return the formatted credentials, excluding defaults. + # Only includes non-empty credential fields to avoid cluttering + # output. + # + # Returns: + # bytes - UTF-8 encoded credential content if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": all_fields = self.model_dump(exclude={"file_type"}) non_default_fields = self.model_dump( @@ -111,6 +132,7 @@ def get_formatted_credentials(self) -> bytes: ) print(f"DEBUG: Cloudflare provider has {len(all_fields)} total fields") print(f"DEBUG: {len(non_default_fields)} non-default fields will be included") + print(f"DEBUG: Excluding empty/default values to minimize credential file") content = "\n".join( f"{key} = {value}" @@ -124,20 +146,26 @@ def get_formatted_credentials(self) -> bytes: return content - # Validate Cloudflare credentials. - # Ensures either API token or email+API key combination is provided. - # Raises: ValueError if neither authentication method is complete @model_validator(mode="after") def validate_cloudflare_credentials(self): + # Validate Cloudflare credentials. + # Ensures either API token or email+API key combination is provided. + # + # Raises: + # ValueError if neither authentication method is complete has_token = bool(self.dns_cloudflare_api_token) has_legacy = bool(self.dns_cloudflare_email and self.dns_cloudflare_api_key) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Cloudflare credential validation:") + print("DEBUG: Cloudflare credential validation:") print(f"DEBUG: API token provided: {has_token}") print(f"DEBUG: Legacy email+key provided: {has_legacy}") + print("DEBUG: At least one authentication method must be complete") if not has_token and not has_legacy: + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print("DEBUG: Neither authentication method is complete") + print("DEBUG: Validation will fail") raise ValueError( "Either 'dns_cloudflare_api_token' or both " "'dns_cloudflare_email' and 'dns_cloudflare_api_key' must be provided." @@ -145,6 +173,8 @@ def validate_cloudflare_credentials(self): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print("DEBUG: Cloudflare credentials validation passed") + auth_method = "API token" if has_token else "email+API key" + print(f"DEBUG: Using {auth_method} authentication method") return self @@ -200,7 +230,8 @@ class DnsMadeEasyProvider(Provider): "dns_dnsmadeeasy_api_key", "dnsmadeeasy_api_key", "api_key" ), "dns_dnsmadeeasy_secret_key": ( - "dns_dnsmadeeasy_secret_key", "dnsmadeeasy_secret_key", "secret_key" + "dns_dnsmadeeasy_secret_key", "dnsmadeeasy_secret_key", + "secret_key" ), } ) @@ -236,15 +267,18 @@ class GoogleProvider(Provider): client_id: str auth_uri: str = "https://accounts.google.com/o/oauth2/auth" token_uri: str = "https://accounts.google.com/o/oauth2/token" - auth_provider_x509_cert_url: str = "https://www.googleapis.com/oauth2/v1/certs" + auth_provider_x509_cert_url: str = ("https://www.googleapis.com/" + "oauth2/v1/certs") client_x509_cert_url: str _validate_aliases = alias_model_validator( { "type": ("type", "google_type", "dns_google_type"), - "project_id": ("project_id", "google_project_id", "dns_google_project_id"), + "project_id": ("project_id", "google_project_id", + "dns_google_project_id"), "private_key_id": ( - "private_key_id", "google_private_key_id", "dns_google_private_key_id" + "private_key_id", "google_private_key_id", + "dns_google_private_key_id" ), "private_key": ( "private_key", "google_private_key", "dns_google_private_key" @@ -252,9 +286,11 @@ class GoogleProvider(Provider): "client_email": ( "client_email", "google_client_email", "dns_google_client_email" ), - "client_id": ("client_id", "google_client_id", "dns_google_client_id"), + "client_id": ("client_id", "google_client_id", + "dns_google_client_id"), "auth_uri": ("auth_uri", "google_auth_uri", "dns_google_auth_uri"), - "token_uri": ("token_uri", "google_token_uri", "dns_google_token_uri"), + "token_uri": ("token_uri", "google_token_uri", + "dns_google_token_uri"), "auth_provider_x509_cert_url": ( "auth_provider_x509_cert_url", "google_auth_provider_x509_cert_url", @@ -268,12 +304,15 @@ class GoogleProvider(Provider): } ) - # Return the formatted credentials in JSON format. - # Google Cloud requires credentials in JSON service account format. - # Returns: bytes - UTF-8 encoded JSON content def get_formatted_credentials(self) -> bytes: + # Return the formatted credentials in JSON format. + # Google Cloud requires credentials in JSON service account format. + # + # Returns: + # bytes - UTF-8 encoded JSON content if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print("DEBUG: Google provider formatting credentials as JSON") + print("DEBUG: Using service account JSON format required by Google Cloud") json_content = self.model_dump_json( indent=2, exclude={"file_type"} @@ -281,14 +320,18 @@ def get_formatted_credentials(self) -> bytes: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Generated {len(json_content)} bytes of JSON credentials") + print("DEBUG: JSON format includes proper indentation for readability") return json_content - # Return the file type that the credentials should be written to. - # Google provider requires JSON format for service account credentials. - # Returns: Literal type indicating JSON file extension @staticmethod def get_file_type() -> Literal["json"]: + # Return the file type that the credentials should be written to. + # Google provider requires JSON format for service account + # credentials. + # + # Returns: + # Literal type indicating JSON file extension return "json" @@ -307,7 +350,8 @@ class InfomaniakProvider(Provider): class IonosProvider(Provider): - # Requires prefix and secret for authentication, with configurable endpoint. + # Requires prefix and secret for authentication, with configurable + # endpoint. dns_ionos_prefix: str dns_ionos_secret: str @@ -317,7 +361,8 @@ class IonosProvider(Provider): { "dns_ionos_prefix": ("dns_ionos_prefix", "ionos_prefix", "prefix"), "dns_ionos_secret": ("dns_ionos_secret", "ionos_secret", "secret"), - "dns_ionos_endpoint": ("dns_ionos_endpoint", "ionos_endpoint", "endpoint"), + "dns_ionos_endpoint": ("dns_ionos_endpoint", "ionos_endpoint", + "endpoint"), } ) @@ -356,7 +401,8 @@ class NSOneProvider(Provider): _validate_aliases = alias_model_validator( { - "dns_nsone_api_key": ("dns_nsone_api_key", "nsone_api_key", "api_key"), + "dns_nsone_api_key": ("dns_nsone_api_key", "nsone_api_key", + "api_key"), } ) @@ -373,7 +419,8 @@ class OvhProvider(Provider): { "dns_ovh_endpoint": ("dns_ovh_endpoint", "ovh_endpoint", "endpoint"), "dns_ovh_application_key": ( - "dns_ovh_application_key", "ovh_application_key", "application_key" + "dns_ovh_application_key", "ovh_application_key", + "application_key" ), "dns_ovh_application_secret": ( "dns_ovh_application_secret", "ovh_application_secret", @@ -399,10 +446,12 @@ class Rfc2136Provider(Provider): _validate_aliases = alias_model_validator( { - "dns_rfc2136_server": ("dns_rfc2136_server", "rfc2136_server", "server"), + "dns_rfc2136_server": ("dns_rfc2136_server", "rfc2136_server", + "server"), "dns_rfc2136_port": ("dns_rfc2136_port", "rfc2136_port", "port"), "dns_rfc2136_name": ("dns_rfc2136_name", "rfc2136_name", "name"), - "dns_rfc2136_secret": ("dns_rfc2136_secret", "rfc2136_secret", "secret"), + "dns_rfc2136_secret": ("dns_rfc2136_secret", "rfc2136_secret", + "secret"), "dns_rfc2136_algorithm": ( "dns_rfc2136_algorithm", "rfc2136_algorithm", "algorithm" ), @@ -412,10 +461,12 @@ class Rfc2136Provider(Provider): } ) - # Return the formatted credentials, excluding defaults. - # RFC2136 provider excludes default values to minimize configuration. - # Returns: bytes - UTF-8 encoded credential content def get_formatted_credentials(self) -> bytes: + # Return the formatted credentials, excluding defaults. + # RFC2136 provider excludes default values to minimize configuration. + # + # Returns: + # bytes - UTF-8 encoded credential content if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": all_fields = self.model_dump(exclude={"file_type"}) non_default_fields = self.model_dump( @@ -423,6 +474,7 @@ def get_formatted_credentials(self) -> bytes: ) print(f"DEBUG: RFC2136 provider has {len(all_fields)} total fields") print(f"DEBUG: {len(non_default_fields)} non-default fields included") + print("DEBUG: Excluding defaults to minimize RFC2136 configuration") content = "\n".join( f"{key} = {value}" @@ -456,13 +508,16 @@ class Route53Provider(Provider): } ) - # Return the formatted credentials in environment variable format. - # Route53 uses environment variables for AWS credentials. - # Returns: bytes - UTF-8 encoded environment variable format def get_formatted_credentials(self) -> bytes: + # Return the formatted credentials in environment variable format. + # Route53 uses environment variables for AWS credentials. + # + # Returns: + # bytes - UTF-8 encoded environment variable format if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": fields = self.model_dump(exclude={"file_type"}) print(f"DEBUG: Route53 provider formatting {len(fields)} fields as env vars") + print("DEBUG: Using environment variable format for AWS credentials") content = "\n".join( f"{key.upper()}={value!r}" @@ -471,14 +526,17 @@ def get_formatted_credentials(self) -> bytes: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Generated {len(content)} bytes of environment variables") + print("DEBUG: Variables will be uppercase as per AWS convention") return content - # Return the file type that the credentials should be written to. - # Route53 provider uses environment variable format. - # Returns: Literal type indicating env file extension @staticmethod def get_file_type() -> Literal["env"]: + # Return the file type that the credentials should be written to. + # Route53 provider uses environment variable format. + # + # Returns: + # Literal type indicating env file extension return "env" @@ -491,10 +549,12 @@ class SakuraCloudProvider(Provider): _validate_aliases = alias_model_validator( { "dns_sakuracloud_api_token": ( - "dns_sakuracloud_api_token", "sakuracloud_api_token", "api_token" + "dns_sakuracloud_api_token", "sakuracloud_api_token", + "api_token" ), "dns_sakuracloud_api_secret": ( - "dns_sakuracloud_api_secret", "sakuracloud_api_secret", "api_secret" + "dns_sakuracloud_api_secret", "sakuracloud_api_secret", + "api_secret" ), } ) @@ -523,7 +583,8 @@ class NjallaProvider(Provider): _validate_aliases = alias_model_validator( { "dns_njalla_token": ( - "dns_njalla_token", "njalla_token", "token", "api_token", "auth_token" + "dns_njalla_token", "njalla_token", "token", "api_token", + "auth_token" ), } ) @@ -537,6 +598,7 @@ class WildcardGenerator: def __init__(self): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print("DEBUG: Initializing WildcardGenerator") + print("DEBUG: Setting up empty domain groups and wildcard storage") # Stores raw domains grouped by identifier self.__domain_groups = {} @@ -545,18 +607,23 @@ def __init__(self): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print("DEBUG: WildcardGenerator initialized with empty groups") - - # Add domains to a group and regenerate wildcards. - # Organizes domains by group and environment for wildcard generation. - # Args: group - Group identifier for these domains - # domains - List of domains to add - # email - Contact email for this domain group - # staging - Whether these domains are for staging environment - def extend(self, group: str, domains: List[str], email: str, staging: bool = False): + print("DEBUG: Ready to accept domain groups for wildcard generation") + + def extend(self, group: str, domains: List[str], email: str, + staging: bool = False): + # Add domains to a group and regenerate wildcards. + # Organizes domains by group and environment for wildcard generation. + # + # Args: + # group: Group identifier for these domains + # domains: List of domains to add + # email: Contact email for this domain group + # staging: Whether these domains are for staging environment if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Extending group '{group}' with {len(domains)} domains") - print(f"DEBUG: Staging mode: {staging}, Email: {email}") - print(f"DEBUG: Domains: {domains}") + print(f"DEBUG: Environment: {'staging' if staging else 'production'}") + print(f"DEBUG: Contact email: {email}") + print(f"DEBUG: Domain list: {domains}") # Initialize group if it doesn't exist if group not in self.__domain_groups: @@ -567,6 +634,7 @@ def extend(self, group: str, domains: List[str], email: str, staging: bool = Fal } if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Created new domain group '{group}'") + print("DEBUG: Group initialized with empty staging and prod sets") # Add domains to appropriate environment env_type = "staging" if staging else "prod" @@ -577,22 +645,26 @@ def extend(self, group: str, domains: List[str], email: str, staging: bool = Fal domains_added += 1 if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added {domains_added} domains to {env_type} environment") + print(f"DEBUG: Added {domains_added} valid domains to {env_type} environment") total_staging = len(self.__domain_groups[group]["staging"]) total_prod = len(self.__domain_groups[group]["prod"]) - print(f"DEBUG: Group '{group}' now has {total_staging} staging, {total_prod} prod domains") + print(f"DEBUG: Group '{group}' totals: {total_staging} staging, {total_prod} prod domains") # Regenerate wildcards after adding new domains self.__generate_wildcards(staging) - # Generate wildcard patterns for the specified environment. - # Creates optimized wildcard certificates that cover multiple subdomains. - # Args: staging - Whether to generate wildcards for staging environment def __generate_wildcards(self, staging: bool = False): + # Generate wildcard patterns for the specified environment. + # Creates optimized wildcard certificates that cover multiple + # subdomains. + # + # Args: + # staging: Whether to generate wildcards for staging environment if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": env_type = "staging" if staging else "prod" print(f"DEBUG: Generating wildcards for {env_type} environment") print(f"DEBUG: Processing {len(self.__domain_groups)} domain groups") + print("DEBUG: Will convert subdomains to wildcard patterns") self.__wildcards.clear() env_type = "staging" if staging else "prod" @@ -614,21 +686,26 @@ def __generate_wildcards(self, staging: bool = False): wildcards_generated += 1 if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated wildcards for {wildcards_generated} domains") + print(f"DEBUG: Generated wildcard patterns for {wildcards_generated} domains") + print("DEBUG: Wildcard generation completed") - # Convert a domain to wildcard patterns and add to the wildcards collection. - # Determines optimal wildcard patterns based on domain structure. - # Args: domain - Domain to process - # group - Group identifier - # env_type - Environment type (staging or prod) def __add_domain_wildcards(self, domain: str, group: str, env_type: str): + # Convert a domain to wildcard patterns and add to the wildcards + # collection. Determines optimal wildcard patterns based on domain + # structure. + # + # Args: + # domain: Domain to process + # group: Group identifier + # env_type: Environment type (staging or prod) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Processing domain '{domain}' for wildcards") + print(f"DEBUG: Processing domain '{domain}' for wildcard patterns") parts = domain.split(".") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Domain has {len(parts)} parts: {parts}") + print("DEBUG: Analyzing domain structure for wildcard generation") # Handle subdomains (domains with more than 2 parts) if len(parts) > 2: @@ -640,20 +717,27 @@ def __add_domain_wildcards(self, domain: str, group: str, env_type: str): self.__wildcards[group][env_type].add(base_domain) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added wildcard '{wildcard_domain}' and base '{base_domain}'") + print(f"DEBUG: Subdomain detected - created wildcard '{wildcard_domain}'") + print(f"DEBUG: Also added base domain '{base_domain}'") + print("DEBUG: Wildcard will cover all subdomains of base") else: # Just add the raw domain for top-level domains self.__wildcards[group][env_type].add(domain) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added top-level domain '{domain}' directly") - - # Get formatted wildcard domains for each group. - # Returns organized wildcard data ready for certificate generation. - # Returns: Dictionary of group data with formatted wildcard domains - def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", "email"], str]]: + print(f"DEBUG: Top-level domain - added '{domain}' directly") + print("DEBUG: No wildcard needed for top-level domain") + + def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", + "email"], str]]: + # Get formatted wildcard domains for each group. + # Returns organized wildcard data ready for certificate generation. + # + # Returns: + # Dictionary of group data with formatted wildcard domains if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Formatting wildcards for {len(self.__wildcards)} groups") + print("DEBUG: Converting wildcard sets to comma-separated strings") result = {} total_domains = 0 @@ -670,87 +754,119 @@ def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", "email"], s if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Group '{group}' {env_type}: {len(domains)} domains") + print(f"DEBUG: Sorted with wildcards first: {sorted_domains[:3]}...") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Formatted {total_domains} total wildcard domains") + print("DEBUG: Ready for certificate generation") return result - # Generate wildcard patterns from a list of domains. - # Static method for generating wildcards without managing groups. - # Args: domains - List of domains to process - # Returns: List of extracted wildcard domains @staticmethod def extract_wildcards_from_domains(domains: List[str]) -> List[str]: + # Generate wildcard patterns from a list of domains. + # Static method for generating wildcards without managing groups. + # + # Args: + # domains: List of domains to process + # + # Returns: + # List of extracted wildcard domains if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Extracting wildcards from {len(domains)} domains") print(f"DEBUG: Input domains: {domains}") + print("DEBUG: Static method - no group management") wildcards = set() for domain in domains: parts = domain.split(".") + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Processing '{domain}' with {len(parts)} parts") + # Generate wildcards for subdomains if len(parts) > 2: base_domain = ".".join(parts[1:]) wildcards.add(f"*.{base_domain}") wildcards.add(base_domain) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Added wildcard *.{base_domain} and base {base_domain}") else: # Just add the domain for top-level domains wildcards.add(domain) + if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Added top-level domain {domain} directly") # Sort with wildcards first result = sorted(wildcards, key=lambda x: x[0] != "*") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(result)} wildcard patterns: {result}") + print(f"DEBUG: Generated {len(result)} wildcard patterns") + print(f"DEBUG: Final result: {result}") return result - # Extract the base domain from a domain name. - # Removes wildcard prefix if present to get the actual domain. - # Args: domain - Input domain name - # Returns: Base domain (without wildcard prefix if present) @staticmethod def get_base_domain(domain: str) -> str: + # Extract the base domain from a domain name. + # Removes wildcard prefix if present to get the actual domain. + # + # Args: + # domain: Input domain name + # + # Returns: + # Base domain (without wildcard prefix if present) base = domain.lstrip("*.") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": if domain != base: - print(f"DEBUG: Extracted base domain '{base}' from '{domain}'") + print(f"DEBUG: Extracted base domain '{base}' from wildcard '{domain}'") + else: + print(f"DEBUG: Domain '{domain}' is already a base domain") return base - # Generate a consistent group name for wildcards. - # Creates a unique identifier for grouping related wildcard certificates. - # Args: domain - The domain name - # provider - DNS provider name or 'http' for HTTP challenge - # challenge_type - Challenge type (dns or http) - # staging - Whether this is for staging environment - # content_hash - Hash of credential content - # profile - Certificate profile (classic, tlsserver or shortlived) - # Returns: A formatted group name string @staticmethod def create_group_name(domain: str, provider: str, challenge_type: str, staging: bool, content_hash: str, profile: str = "classic") -> str: + # Generate a consistent group name for wildcards. + # Creates a unique identifier for grouping related wildcard + # certificates. + # + # Args: + # domain: The domain name + # provider: DNS provider name or 'http' for HTTP challenge + # challenge_type: Challenge type (dns or http) + # staging: Whether this is for staging environment + # content_hash: Hash of credential content + # profile: Certificate profile (classic, tlsserver or shortlived) + # + # Returns: + # A formatted group name string if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": print(f"DEBUG: Creating group name for domain '{domain}'") print(f"DEBUG: Provider: {provider}, Challenge: {challenge_type}") - print(f"DEBUG: Staging: {staging}, Profile: {profile}") - print(f"DEBUG: Content hash: {content_hash[:10]}...") + print(f"DEBUG: Environment: {'staging' if staging else 'production'}") + print(f"DEBUG: Profile: {profile}") + print(f"DEBUG: Content hash: {content_hash[:10]}... (truncated)") # Extract base domain and format it for the group name - base_domain = WildcardGenerator.get_base_domain(domain).replace(".", "-") + base_domain = WildcardGenerator.get_base_domain(domain).replace(".", + "-") env = "staging" if staging else "prod" # Use provider name for DNS challenge, otherwise use 'http' challenge_identifier = provider if challenge_type == "dns" else "http" - group_name = f"{challenge_identifier}_{env}_{profile}_{base_domain}_{content_hash}" + group_name = (f"{challenge_identifier}_{env}_{profile}_{base_domain}_" + f"{content_hash}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + print(f"DEBUG: Base domain formatted: {base_domain}") + print(f"DEBUG: Challenge identifier: {challenge_identifier}") print(f"DEBUG: Generated group name: '{group_name}'") + print("DEBUG: Group name ensures consistent certificate grouping") return group_name \ No newline at end of file From ca73a5c75b9079b1e34663c64881bd374574f1a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 09:14:53 +0200 Subject: [PATCH 05/15] update formating, documentation of functions, add more debugging output - formating: apply 79-character line limits for code (120 max fallback) - documentation: aff documentation of functions - debugging: add debugging output when LOG_LEVEL: "DEBUG" --- src/common/core/letsencrypt/ui/actions.py | 437 +++++- .../letsencrypt/ui/blueprints/letsencrypt.py | 422 +++++- .../ui/blueprints/static/js/main.js | 1325 ++++++++++------- src/common/core/letsencrypt/ui/hooks.py | 84 +- 4 files changed, 1577 insertions(+), 691 deletions(-) diff --git a/src/common/core/letsencrypt/ui/actions.py b/src/common/core/letsencrypt/ui/actions.py index b1d0ba9bbe..e8668caa16 100644 --- a/src/common/core/letsencrypt/ui/actions.py +++ b/src/common/core/letsencrypt/ui/actions.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - from io import BytesIO from logging import getLogger +from os import getenv from os.path import sep from pathlib import Path from shutil import rmtree @@ -16,18 +15,131 @@ def extract_cache(folder_path, cache_files): + # Extract Let's Encrypt cache files to specified folder path. + # + # Args: + # folder_path (Path): Destination folder path for extraction + # cache_files (list): List of cache file dictionaries containing + # file data to extract + logger = getLogger("UI") + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + logger.debug(f"Starting cache extraction to {folder_path}") + logger.debug(f"Processing {len(cache_files)} cache files") + logger.debug(f"Target folder exists: {folder_path.exists()}") + folder_path.mkdir(parents=True, exist_ok=True) + + if is_debug: + logger.debug(f"Created directory structure: {folder_path}") + logger.debug(f"Directory permissions: {oct(folder_path.stat().st_mode)}") - for cache_file in cache_files: - if cache_file["file_name"].endswith(".tgz") and cache_file["file_name"].startswith("folder:"): - with tar_open(fileobj=BytesIO(cache_file["data"]), mode="r:gz") as tar: - try: - tar.extractall(folder_path, filter="fully_trusted") - except TypeError: - tar.extractall(folder_path) + extracted_files = 0 + total_bytes = 0 + + for i, cache_file in enumerate(cache_files): + file_name = cache_file.get("file_name", "unknown") + file_data = cache_file.get("data", b"") + + if is_debug: + logger.debug(f"Examining cache file {i+1}/{len(cache_files)}: {file_name}") + logger.debug(f"File size: {len(file_data)} bytes") + + if (cache_file["file_name"].endswith(".tgz") and + cache_file["file_name"].startswith("folder:")): + + if is_debug: + logger.debug(f"Processing archive: {cache_file['file_name']}") + logger.debug(f"Archive size: {len(cache_file['data'])} bytes") + + try: + with tar_open(fileobj=BytesIO(cache_file["data"]), + mode="r:gz") as tar: + + members = tar.getmembers() + if is_debug: + logger.debug(f"Archive contains {len(members)} members") + # Show first few members + for j, member in enumerate(members[:5]): + logger.debug(f" Member {j+1}: {member.name} " + f"({member.size} bytes, " + f"{'dir' if member.isdir() else 'file'})") + if len(members) > 5: + logger.debug(f" ... and {len(members) - 5} more members") + + try: + tar.extractall(folder_path, filter="fully_trusted") + if is_debug: + logger.debug("Extraction completed with fully_trusted filter") + except TypeError: + # Fallback for older Python versions without filter + if is_debug: + logger.debug("Using fallback extraction without filter") + tar.extractall(folder_path) + + extracted_files += 1 + total_bytes += len(cache_file['data']) + + if is_debug: + logger.debug(f"Successfully extracted {cache_file['file_name']}") + logger.debug(f"Extracted {len(members)} items from archive") + + except Exception as e: + logger.error(f"Failed to extract {cache_file['file_name']}: {e}") + if is_debug: + logger.debug(f"Extraction error details: {format_exc()}") + else: + if is_debug: + logger.debug(f"Skipping non-archive file: {file_name}") + if is_debug: + logger.debug(f"Cache extraction completed:") + logger.debug(f" - Files processed: {len(cache_files)}") + logger.debug(f" - Archives extracted: {extracted_files}") + logger.debug(f" - Total bytes processed: {total_bytes}") + + # List final directory contents + if folder_path.exists(): + all_items = list(folder_path.rglob("*")) + files = [item for item in all_items if item.is_file()] + dirs = [item for item in all_items if item.is_dir()] + + logger.debug(f"Final directory structure:") + logger.debug(f" - Total items: {len(all_items)}") + logger.debug(f" - Files: {len(files)}") + logger.debug(f" - Directories: {len(dirs)}") + + # Show some example files + for i, file_item in enumerate(files[:5]): + rel_path = file_item.relative_to(folder_path) + logger.debug(f" File {i+1}: {rel_path} ({file_item.stat().st_size} bytes)") + if len(files) > 5: + logger.debug(f" ... and {len(files) - 5} more files") -def retrieve_certificates_info(folder_paths: Tuple[Path, Path]) -> dict: + +def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: + # Retrieve comprehensive certificate information from folder paths. + # + # Parses Let's Encrypt certificate files and renewal configurations + # to extract detailed certificate information including validity dates, + # issuer information, and configuration details. + # + # Args: + # folder_paths (Tuple[Path, ...]): Tuple of folder paths containing + # certificate data to process + # + # Returns: + # dict: Dictionary containing lists of certificate information + # with keys for domain, common_name, issuer, validity dates, + # and other certificate metadata + logger = getLogger("UI") + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + logger.debug(f"Retrieving certificate info from {len(folder_paths)} " + f"folder paths") + certificates = { "domain": [], "common_name": [], @@ -43,12 +155,29 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, Path]) -> dict: "key_type": [], } - for folder_path in folder_paths: - for cert_file in folder_path.joinpath("live").glob("*/fullchain.pem"): + total_certs_processed = 0 + + for folder_idx, folder_path in enumerate(folder_paths): + if is_debug: + logger.debug(f"Processing folder {folder_idx + 1}/{len(folder_paths)}: " + f"{folder_path}") + + cert_files = list(folder_path.joinpath("live").glob("*/fullchain.pem")) + + if is_debug: + logger.debug(f"Found {len(cert_files)} certificate files in " + f"{folder_path}") + + for cert_file in cert_files: domain = cert_file.parent.name certificates["domain"].append(domain) + total_certs_processed += 1 + + if is_debug: + logger.debug(f"Processing certificate {total_certs_processed}: " + f"{domain}") - # Default values + # Initialize default certificate information cert_info = { "common_name": "Unknown", "issuer": "Unknown", @@ -64,63 +193,190 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, Path]) -> dict: "key_type": "Unknown", } - # * Parsing the certificate + # Parse the certificate file try: - cert = x509.load_pem_x509_certificate(cert_file.read_bytes(), default_backend()) + if is_debug: + logger.debug(f"Loading X.509 certificate from {cert_file}") + logger.debug(f"Certificate file size: {cert_file.stat().st_size} bytes") + + cert_bytes = cert_file.read_bytes() + if is_debug: + logger.debug(f"Read {len(cert_bytes)} bytes from certificate file") + logger.debug(f"Certificate data preview: {cert_bytes[:100]}...") + + cert = x509.load_pem_x509_certificate( + cert_bytes, default_backend() + ) + + if is_debug: + logger.debug(f"Successfully loaded certificate for {domain}") + logger.debug(f"Certificate version: {cert.version}") + logger.debug(f"Certificate serial: {cert.serial_number}") - # ? Getting the subject - subject = cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME) + # Extract subject (Common Name) + subject = cert.subject.get_attributes_for_oid( + x509.NameOID.COMMON_NAME + ) if subject: cert_info["common_name"] = subject[0].value + if is_debug: + logger.debug(f"Certificate CN: {cert_info['common_name']}") + else: + if is_debug: + logger.debug("No Common Name found in certificate subject") + logger.debug(f"Full subject: {cert.subject}") - # ? Getting the issuer - issuer = cert.issuer.get_attributes_for_oid(x509.NameOID.COMMON_NAME) + # Extract issuer (Certificate Authority) + issuer = cert.issuer.get_attributes_for_oid( + x509.NameOID.COMMON_NAME + ) if issuer: cert_info["issuer"] = issuer[0].value + if is_debug: + logger.debug(f"Certificate issuer: {cert_info['issuer']}") + else: + if is_debug: + logger.debug("No Common Name found in certificate issuer") + logger.debug(f"Full issuer: {cert.issuer}") - # ? Getting the validity period - cert_info["valid_from"] = cert.not_valid_before.strftime("%d-%m-%Y %H:%M:%S UTC") - cert_info["valid_to"] = cert.not_valid_after.strftime("%d-%m-%Y %H:%M:%S UTC") + # Extract validity period + cert_info["valid_from"] = ( + cert.not_valid_before.strftime("%d-%m-%Y %H:%M:%S UTC") + ) + cert_info["valid_to"] = ( + cert.not_valid_after.strftime("%d-%m-%Y %H:%M:%S UTC") + ) + + if is_debug: + logger.debug(f"Certificate validity: {cert_info['valid_from']} " + f"to {cert_info['valid_to']}") + # Check if certificate is currently valid + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + is_valid = cert.not_valid_before <= now <= cert.not_valid_after + logger.debug(f"Certificate currently valid: {is_valid}") - # ? Getting the serial number + # Extract serial number cert_info["serial_number"] = str(cert.serial_number) - # ? Getting the fingerprint - cert_info["fingerprint"] = cert.fingerprint(hashes.SHA256()).hex() + # Calculate fingerprint + fingerprint_bytes = cert.fingerprint(hashes.SHA256()) + cert_info["fingerprint"] = fingerprint_bytes.hex() + + if is_debug: + logger.debug(f"Certificate fingerprint: {cert_info['fingerprint']}") - # ? Getting the version + # Extract version cert_info["version"] = cert.version.name - except BaseException: - print(f"Error while parsing certificate {cert_file}: {format_exc()}", flush=True) + + if is_debug: + logger.debug(f"Certificate processing completed for {domain}") + logger.debug(f" - Serial: {cert_info['serial_number']}") + logger.debug(f" - Version: {cert_info['version']}") + logger.debug(f" - Subject: {cert_info['common_name']}") + logger.debug(f" - Issuer: {cert_info['issuer']}") + + except BaseException as e: + error_msg = f"Error while parsing certificate {cert_file}: {e}" + logger.error(error_msg) + if is_debug: + logger.debug(f"Certificate parsing error details:") + logger.debug(f" - Error type: {type(e).__name__}") + logger.debug(f" - Error message: {str(e)}") + logger.debug(f" - Full traceback: {format_exc()}") + logger.debug(f" - Certificate file: {cert_file}") + logger.debug(f" - File exists: {cert_file.exists()}") + logger.debug(f" - File readable: {cert_file.is_file()}") - # * Parsing the renewal configuration + # Parse the renewal configuration file try: renewal_file = folder_path.joinpath("renewal", f"{domain}.conf") + if renewal_file.exists(): + if is_debug: + logger.debug(f"Processing renewal configuration: " + f"{renewal_file}") + with renewal_file.open("r") as f: - for line in f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("preferred_profile = "): - cert_info["preferred_profile"] = line.split(" = ")[1].strip() + cert_info["preferred_profile"] = ( + line.split(" = ")[1].strip() + ) elif line.startswith("pref_challs = "): - cert_info["challenge"] = line.split(" = ")[1].strip().split(",")[0] + # Take first challenge from comma-separated list + challenges = line.split(" = ")[1].strip() + cert_info["challenge"] = challenges.split(",")[0] elif line.startswith("authenticator = "): - cert_info["authenticator"] = line.split(" = ")[1].strip() + cert_info["authenticator"] = ( + line.split(" = ")[1].strip() + ) elif line.startswith("server = "): - cert_info["issuer_server"] = line.split(" = ")[1].strip() + cert_info["issuer_server"] = ( + line.split(" = ")[1].strip() + ) elif line.startswith("key_type = "): - cert_info["key_type"] = line.split(" = ")[1].strip() - except BaseException: - print(f"Error while parsing renewal configuration {renewal_file}: {format_exc()}", flush=True) + cert_info["key_type"] = ( + line.split(" = ")[1].strip() + ) + + if is_debug: + logger.debug(f"Renewal config parsed - Profile: " + f"{cert_info['preferred_profile']}, " + f"Challenge: {cert_info['challenge']}, " + f"Key type: {cert_info['key_type']}") + else: + if is_debug: + logger.debug(f"No renewal configuration found for " + f"{domain}") + + except BaseException as e: + error_msg = (f"Error while parsing renewal configuration " + f"{renewal_file}: {e}") + logger.error(error_msg) + if is_debug: + logger.debug(f"Renewal config parsing error: {format_exc()}") - # Append values to corresponding lists in certificates dictionary + # Append all certificate information to the results for key in cert_info: certificates[key].append(cert_info[key]) + if is_debug: + logger.debug(f"Certificate retrieval complete. Processed " + f"{total_certs_processed} certificates from " + f"{len(folder_paths)} folders") + return certificates def pre_render(app, *args, **kwargs): + # Pre-render function to prepare Let's Encrypt certificate data for UI. + # + # Retrieves certificate information from database cache files and + # prepares the data structure for rendering in the web interface. + # Handles extraction of cache files, certificate parsing, and error + # handling for the certificate management interface. + # + # Args: + # app: Flask application instance + # *args: Variable length argument list + # **kwargs: Keyword arguments containing database connection and + # other configuration options + # + # Returns: + # dict: Dictionary containing certificate data, display configuration, + # and any error information for the UI template logger = getLogger("UI") + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + logger.debug("Starting pre-render for Let's Encrypt certificates") + + # Initialize return structure with default values ret = { "list_certificates": { "data": { @@ -148,23 +404,106 @@ def pre_render(app, *args, **kwargs): } root_folder = Path(sep, "var", "tmp", "bunkerweb", "ui") + folder_path = None + try: - # ? Fetching Let's Encrypt cache files - regular_cache_files = kwargs["db"].get_jobs_cache_files(job_name="certbot-renew") + if is_debug: + logger.debug("Starting Let's Encrypt data retrieval process") + logger.debug(f"Database connection available: {'db' in kwargs}") + logger.debug(f"Root folder: {root_folder}") + + # Retrieve cache files from database + if is_debug: + logger.debug("Fetching cache files from database for job: certbot-renew") + + regular_cache_files = kwargs["db"].get_jobs_cache_files( + job_name="certbot-renew" + ) + + if is_debug: + logger.debug(f"Retrieved {len(regular_cache_files)} cache files") + for i, cache_file in enumerate(regular_cache_files): + file_name = cache_file.get("file_name", "unknown") + file_size = len(cache_file.get("data", b"")) + logger.debug(f" Cache file {i+1}: {file_name} ({file_size} bytes)") - # ? Extracting cache files - folder_path = root_folder.joinpath("letsencrypt", str(uuid4())) + # Create unique temporary folder for extraction + folder_uuid = str(uuid4()) + folder_path = root_folder.joinpath("letsencrypt", folder_uuid) regular_le_folder = folder_path.joinpath("regular") + + if is_debug: + logger.debug(f"Using temporary folder UUID: {folder_uuid}") + logger.debug(f"Temporary folder path: {folder_path}") + logger.debug(f"Regular LE folder: {regular_le_folder}") + + # Extract cache files to temporary location + if is_debug: + logger.debug("Starting cache file extraction") + extract_cache(regular_le_folder, regular_cache_files) + + if is_debug: + logger.debug("Cache extraction completed, starting certificate parsing") - # ? We retrieve the certificates from the cache files by parsing the content of the .pem files - ret["list_certificates"]["data"] = retrieve_certificates_info((regular_le_folder,)) + # Parse certificates and retrieve information + cert_data = retrieve_certificates_info((regular_le_folder,)) + + cert_count = len(cert_data.get("domain", [])) + + if is_debug: + logger.debug(f"Certificate parsing completed") + logger.debug(f"Total certificates processed: {cert_count}") + logger.debug(f"Certificate data keys: {list(cert_data.keys())}") + + # Log sample certificate data (first certificate if available) + if cert_count > 0: + logger.debug("Sample certificate data (first certificate):") + for key in cert_data: + value = cert_data[key][0] if cert_data[key] else "None" + logger.debug(f" {key}: {value}") + + ret["list_certificates"]["data"] = cert_data + + logger.info(f"Pre-render completed successfully with {cert_count} " + f"certificates") + + if is_debug: + logger.debug(f"Return data structure keys: {list(ret.keys())}") + logger.debug(f"Certificate list structure: {list(ret['list_certificates'].keys())}") + except BaseException as e: - logger.debug(format_exc()) - logger.error(f"Failed to get Let's Encrypt certificates: {e}") + error_msg = f"Failed to get Let's Encrypt certificates: {e}" + logger.error(error_msg) + + if is_debug: + logger.debug(f"Pre-render error occurred:") + logger.debug(f" - Error type: {type(e).__name__}") + logger.debug(f" - Error message: {str(e)}") + logger.debug(f" - Error traceback: {format_exc()}") + logger.debug(f" - kwargs keys: {list(kwargs.keys()) if kwargs else 'None'}") + if "db" in kwargs: + logger.debug(f" - Database object type: {type(kwargs['db'])}") + ret["error"] = str(e) + finally: - if folder_path: - rmtree(root_folder, ignore_errors=True) + # Clean up temporary files + if folder_path and folder_path.exists(): + try: + if is_debug: + logger.debug(f"Cleaning up temporary folder: {root_folder}") + + rmtree(root_folder, ignore_errors=True) + + if is_debug: + logger.debug("Temporary folder cleanup completed") + + except Exception as cleanup_error: + logger.warning(f"Failed to clean up temporary folder " + f"{root_folder}: {cleanup_error}") + + if is_debug: + logger.debug("Pre-render function completed") - return ret + return ret \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py index 3ae7764062..e42777b363 100644 --- a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py +++ b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py @@ -26,7 +26,9 @@ template_folder=f"{blueprint_path}/templates", ) -CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot") +CERTBOT_BIN = join( + sep, "usr", "share", "bunkerweb", "deps", "python", "bin", "certbot" +) LE_CACHE_DIR = join(sep, "var", "cache", "bunkerweb", "letsencrypt", "etc") DATA_PATH = join(sep, "var", "tmp", "bunkerweb", "ui", "letsencrypt", "etc") WORK_DIR = join(sep, "var", "tmp", "bunkerweb", "ui", "letsencrypt", "lib") @@ -36,21 +38,101 @@ def download_certificates(): - rmtree(DATA_PATH, ignore_errors=True) + # Download and extract Let's Encrypt certificates from database cache. + # + # Retrieves certificate cache files from the database and extracts them + # to the local data path for processing. + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + LOGGER.debug(f"Starting certificate download process") + LOGGER.debug(f"Target directory: {DATA_PATH}") + LOGGER.debug(f"Cache directory: {LE_CACHE_DIR}") + + # Clean up and create fresh directory + if Path(DATA_PATH).exists(): + if is_debug: + LOGGER.debug(f"Removing existing directory: {DATA_PATH}") + rmtree(DATA_PATH, ignore_errors=True) + + if is_debug: + LOGGER.debug(f"Creating directory structure: {DATA_PATH}") Path(DATA_PATH).mkdir(parents=True, exist_ok=True) + if is_debug: + LOGGER.debug("Fetching cache files from database") cache_files = DB.get_jobs_cache_files(job_name="certbot-renew") - + + if is_debug: + LOGGER.debug(f"Retrieved {len(cache_files)} cache files") + for i, cache_file in enumerate(cache_files): + LOGGER.debug(f"Cache file {i+1}: {cache_file['file_name']} " + f"({len(cache_file.get('data', b''))} bytes)") + + extracted_count = 0 for cache_file in cache_files: - if cache_file["file_name"].endswith(".tgz") and cache_file["file_name"].startswith("folder:"): - with tar_open(fileobj=BytesIO(cache_file["data"]), mode="r:gz") as tar: - try: - tar.extractall(DATA_PATH, filter="fully_trusted") - except TypeError: - tar.extractall(DATA_PATH) + if (cache_file["file_name"].endswith(".tgz") and + cache_file["file_name"].startswith("folder:")): + + if is_debug: + LOGGER.debug(f"Extracting cache file: {cache_file['file_name']}") + LOGGER.debug(f"File size: {len(cache_file['data'])} bytes") + + try: + with tar_open(fileobj=BytesIO(cache_file["data"]), + mode="r:gz") as tar: + member_count = len(tar.getmembers()) + if is_debug: + LOGGER.debug(f"Archive contains {member_count} members") + + try: + tar.extractall(DATA_PATH, filter="fully_trusted") + if is_debug: + LOGGER.debug("Extraction completed with fully_trusted filter") + except TypeError: + if is_debug: + LOGGER.debug("Falling back to extraction without filter") + tar.extractall(DATA_PATH) + + extracted_count += 1 + if is_debug: + LOGGER.debug(f"Successfully extracted {cache_file['file_name']}") + + except Exception as e: + LOGGER.error(f"Failed to extract {cache_file['file_name']}: {e}") + if is_debug: + LOGGER.debug(f"Extraction error details: {format_exc()}") + else: + if is_debug: + LOGGER.debug(f"Skipping non-matching file: {cache_file['file_name']}") + + if is_debug: + LOGGER.debug(f"Certificate download completed: {extracted_count} files extracted") + # List extracted directory contents + if Path(DATA_PATH).exists(): + contents = list(Path(DATA_PATH).rglob("*")) + LOGGER.debug(f"Extracted directory contains {len(contents)} items") + for item in contents[:10]: # Show first 10 items + LOGGER.debug(f" - {item}") + if len(contents) > 10: + LOGGER.debug(f" ... and {len(contents) - 10} more items") def retrieve_certificates(): + # Retrieve and parse Let's Encrypt certificate information. + # + # Downloads certificates from cache and parses both the certificate + # files and renewal configuration to extract comprehensive certificate + # information. + # + # Returns: + # dict: Dictionary containing lists of certificate information + # including domain, issuer, validity dates, etc. + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + LOGGER.debug("Starting certificate retrieval") + download_certificates() certificates = { @@ -69,9 +151,20 @@ def retrieve_certificates(): "key_type": [], } - for cert_file in Path(DATA_PATH).joinpath("live").glob("*/fullchain.pem"): + cert_files = list(Path(DATA_PATH).joinpath("live").glob("*/fullchain.pem")) + + if is_debug: + LOGGER.debug(f"Processing {len(cert_files)} certificate files") + + for cert_file in cert_files: domain = cert_file.parent.name certificates["domain"].append(domain) + + if is_debug: + LOGGER.debug(f"Processing certificate {len(certificates['domain'])}: {domain}") + LOGGER.debug(f"Certificate file path: {cert_file}") + LOGGER.debug(f"Certificate file size: {cert_file.stat().st_size} bytes") + cert_info = { "common_name": "Unknown", "issuer": "Unknown", @@ -86,51 +179,164 @@ def retrieve_certificates(): "authenticator": "Unknown", "key_type": "Unknown", } + try: - cert = x509.load_pem_x509_certificate(cert_file.read_bytes(), default_backend()) - subject = cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME) + if is_debug: + LOGGER.debug(f"Loading X.509 certificate from {cert_file}") + + cert_data = cert_file.read_bytes() + if is_debug: + LOGGER.debug(f"Certificate data length: {len(cert_data)} bytes") + LOGGER.debug(f"Certificate starts with: {cert_data[:50]}") + + cert = x509.load_pem_x509_certificate( + cert_data, default_backend() + ) + + if is_debug: + LOGGER.debug(f"Successfully loaded certificate for {domain}") + LOGGER.debug(f"Certificate subject: {cert.subject}") + LOGGER.debug(f"Certificate issuer: {cert.issuer}") + + subject = cert.subject.get_attributes_for_oid( + x509.NameOID.COMMON_NAME + ) if subject: cert_info["common_name"] = subject[0].value - issuer = cert.issuer.get_attributes_for_oid(x509.NameOID.COMMON_NAME) + if is_debug: + LOGGER.debug(f"Certificate CN extracted: {cert_info['common_name']}") + else: + if is_debug: + LOGGER.debug("No Common Name found in certificate subject") + + issuer = cert.issuer.get_attributes_for_oid( + x509.NameOID.COMMON_NAME + ) if issuer: cert_info["issuer"] = issuer[0].value - cert_info["valid_from"] = cert.not_valid_before.astimezone().isoformat() - cert_info["valid_to"] = cert.not_valid_after.astimezone().isoformat() + if is_debug: + LOGGER.debug(f"Certificate issuer extracted: {cert_info['issuer']}") + else: + if is_debug: + LOGGER.debug("No Common Name found in certificate issuer") + + cert_info["valid_from"] = ( + cert.not_valid_before.astimezone().isoformat() + ) + cert_info["valid_to"] = ( + cert.not_valid_after.astimezone().isoformat() + ) + + if is_debug: + LOGGER.debug(f"Certificate validity period: " + f"{cert_info['valid_from']} to {cert_info['valid_to']}") + cert_info["serial_number"] = str(cert.serial_number) cert_info["fingerprint"] = cert.fingerprint(hashes.SHA256()).hex() cert_info["version"] = cert.version.name + + if is_debug: + LOGGER.debug(f"Certificate details extracted:") + LOGGER.debug(f" - Serial: {cert_info['serial_number']}") + LOGGER.debug(f" - Fingerprint: {cert_info['fingerprint'][:16]}...") + LOGGER.debug(f" - Version: {cert_info['version']}") + except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while parsing certificate {cert_file}: {e}") + if is_debug: + LOGGER.debug(f"Certificate parsing failed for {domain}: {str(e)}") + LOGGER.debug(f"Error type: {type(e).__name__}") try: renewal_file = Path(DATA_PATH).joinpath("renewal", f"{domain}.conf") if renewal_file.exists(): + if is_debug: + LOGGER.debug(f"Processing renewal file: {renewal_file}") + LOGGER.debug(f"Renewal file size: {renewal_file.stat().st_size} bytes") + + config_lines_processed = 0 with renewal_file.open("r") as f: - for line in f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line or line.startswith("#"): + continue + + config_lines_processed += 1 + if is_debug and line_num <= 10: # Debug first 10 meaningful lines + LOGGER.debug(f"Renewal config line {line_num}: {line}") + if line.startswith("preferred_profile = "): - cert_info["preferred_profile"] = line.split(" = ")[1].strip() + cert_info["preferred_profile"] = ( + line.split(" = ")[1].strip() + ) + if is_debug: + LOGGER.debug(f"Found preferred_profile: {cert_info['preferred_profile']}") elif line.startswith("pref_challs = "): - cert_info["challenge"] = line.split(" = ")[1].strip().split(",")[0] + challenges = line.split(" = ")[1].strip() + cert_info["challenge"] = challenges.split(",")[0] + if is_debug: + LOGGER.debug(f"Found challenge: {cert_info['challenge']} (from {challenges})") elif line.startswith("authenticator = "): - cert_info["authenticator"] = line.split(" = ")[1].strip() + cert_info["authenticator"] = ( + line.split(" = ")[1].strip() + ) + if is_debug: + LOGGER.debug(f"Found authenticator: {cert_info['authenticator']}") elif line.startswith("server = "): - cert_info["issuer_server"] = line.split(" = ")[1].strip() + cert_info["issuer_server"] = ( + line.split(" = ")[1].strip() + ) + if is_debug: + LOGGER.debug(f"Found issuer_server: {cert_info['issuer_server']}") elif line.startswith("key_type = "): - cert_info["key_type"] = line.split(" = ")[1].strip() + cert_info["key_type"] = ( + line.split(" = ")[1].strip() + ) + if is_debug: + LOGGER.debug(f"Found key_type: {cert_info['key_type']}") + + if is_debug: + LOGGER.debug(f"Processed {config_lines_processed} configuration lines") + LOGGER.debug(f"Final renewal configuration for {domain}:") + LOGGER.debug(f" - Profile: {cert_info['preferred_profile']}") + LOGGER.debug(f" - Challenge: {cert_info['challenge']}") + LOGGER.debug(f" - Authenticator: {cert_info['authenticator']}") + LOGGER.debug(f" - Server: {cert_info['issuer_server']}") + LOGGER.debug(f" - Key type: {cert_info['key_type']}") + else: + if is_debug: + LOGGER.debug(f"No renewal file found for {domain} at {renewal_file}") + except BaseException as e: LOGGER.debug(format_exc()) - LOGGER.error(f"Error while parsing renewal configuration {renewal_file}: {e}") + LOGGER.error(f"Error while parsing renewal configuration " + f"{renewal_file}: {e}") + if is_debug: + LOGGER.debug(f"Renewal config parsing failed for {domain}: {str(e)}") + LOGGER.debug(f"Error type: {type(e).__name__}") for key in cert_info: certificates[key].append(cert_info[key]) + if is_debug: + LOGGER.debug(f"Retrieved {len(certificates['domain'])} certificates") + return certificates @letsencrypt.route("/letsencrypt", methods=["GET"]) @login_required def letsencrypt_page(): + # Render the Let's Encrypt certificates management page. + # + # Returns: + # str: Rendered HTML template for the Let's Encrypt page + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + LOGGER.debug("Rendering Let's Encrypt page") + return render_template("letsencrypt.html") @@ -138,55 +344,100 @@ def letsencrypt_page(): @login_required @cors_required def letsencrypt_fetch(): + # Fetch certificate data for DataTables AJAX requests. + # + # Retrieves and formats certificate information for display in the + # DataTables interface. + # + # Returns: + # dict: JSON response containing certificate data, record counts, + # and draw number for DataTables + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + LOGGER.debug("Fetching certificates for DataTables") + cert_list = [] try: certs = retrieve_certificates() - LOGGER.debug(f"Certificates: {certs}") + + if is_debug: + LOGGER.debug(f"Retrieved certificates: {len(certs.get('domain', []))}") + for i, domain in enumerate(certs.get("domain", [])): - cert_list.append( - { - "domain": domain, - "common_name": certs.get("common_name", [""])[i], - "issuer": certs.get("issuer", [""])[i], - "issuer_server": certs.get("issuer_server", [""])[i], - "valid_from": certs.get("valid_from", [""])[i], - "valid_to": certs.get("valid_to", [""])[i], - "serial_number": certs.get("serial_number", [""])[i], - "fingerprint": certs.get("fingerprint", [""])[i], - "version": certs.get("version", [""])[i], - "preferred_profile": certs.get("preferred_profile", [""])[i], - "challenge": certs.get("challenge", [""])[i], - "authenticator": certs.get("authenticator", [""])[i], - "key_type": certs.get("key_type", [""])[i], - } - ) + cert_data = { + "domain": domain, + "common_name": certs.get("common_name", [""])[i], + "issuer": certs.get("issuer", [""])[i], + "issuer_server": certs.get("issuer_server", [""])[i], + "valid_from": certs.get("valid_from", [""])[i], + "valid_to": certs.get("valid_to", [""])[i], + "serial_number": certs.get("serial_number", [""])[i], + "fingerprint": certs.get("fingerprint", [""])[i], + "version": certs.get("version", [""])[i], + "preferred_profile": certs.get("preferred_profile", [""])[i], + "challenge": certs.get("challenge", [""])[i], + "authenticator": certs.get("authenticator", [""])[i], + "key_type": certs.get("key_type", [""])[i], + } + cert_list.append(cert_data) + + if is_debug: + LOGGER.debug(f"Added certificate to list: {domain}") + except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while fetching certificates: {e}") - return jsonify( - { - "data": cert_list, - "recordsTotal": len(cert_list), - "recordsFiltered": len(cert_list), - "draw": int(request.form.get("draw", 1)), - } - ) + response_data = { + "data": cert_list, + "recordsTotal": len(cert_list), + "recordsFiltered": len(cert_list), + "draw": int(request.form.get("draw", 1)), + } + + if is_debug: + LOGGER.debug(f"Returning {len(cert_list)} certificates to DataTables") + + return jsonify(response_data) @letsencrypt.route("/letsencrypt/delete", methods=["POST"]) @login_required @cors_required def letsencrypt_delete(): + # Delete a Let's Encrypt certificate. + # + # Removes the specified certificate using certbot and cleans up + # associated files and directories. Updates the database cache + # with the modified certificate data. + # + # Returns: + # dict: JSON response indicating success or failure of the + # deletion operation + is_debug = getenv("LOG_LEVEL") == "debug" + cert_name = request.json.get("cert_name") if not cert_name: + if is_debug: + LOGGER.debug("Certificate deletion request missing cert_name") return jsonify({"status": "ko", "message": "Missing cert_name"}), 400 + if is_debug: + LOGGER.debug(f"Starting deletion of certificate: {cert_name}") + download_certificates() env = {"PATH": getenv("PATH", ""), "PYTHONPATH": getenv("PYTHONPATH", "")} - env["PYTHONPATH"] = env["PYTHONPATH"] + (f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "") + env["PYTHONPATH"] = env["PYTHONPATH"] + ( + f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" + ) + + if is_debug: + LOGGER.debug(f"Running certbot delete for {cert_name}") + LOGGER.debug(f"Environment: PATH={env['PATH'][:100]}...") + LOGGER.debug(f"PYTHONPATH: {env['PYTHONPATH'][:100]}...") delete_proc = run( [ @@ -210,18 +461,30 @@ def letsencrypt_delete(): check=False, ) + if is_debug: + LOGGER.debug(f"Certbot delete return code: {delete_proc.returncode}") + if delete_proc.stdout: + LOGGER.debug(f"Certbot output: {delete_proc.stdout}") + if delete_proc.returncode == 0: LOGGER.info(f"Successfully deleted certificate {cert_name}") + + # Clean up certificate directories and files cert_dir = Path(DATA_PATH).joinpath("live", cert_name) archive_dir = Path(DATA_PATH).joinpath("archive", cert_name) renewal_file = Path(DATA_PATH).joinpath("renewal", f"{cert_name}.conf") + if is_debug: + LOGGER.debug(f"Cleaning up directories for {cert_name}") + for path in (cert_dir, archive_dir): if path.exists(): try: for file in path.glob("*"): try: file.unlink() + if is_debug: + LOGGER.debug(f"Removed file: {file}") except Exception as e: LOGGER.error(f"Failed to remove file {file}: {e}") path.rmdir() @@ -233,36 +496,71 @@ def letsencrypt_delete(): try: renewal_file.unlink() LOGGER.info(f"Removed renewal file {renewal_file}") + if is_debug: + LOGGER.debug(f"Renewal file removed: {renewal_file}") except Exception as e: - LOGGER.error(f"Failed to remove renewal file {renewal_file}: {e}") + LOGGER.error(f"Failed to remove renewal file " + f"{renewal_file}: {e}") + # Update database cache with modified certificate data try: + if is_debug: + LOGGER.debug("Updating database cache with modified data") + dir_path = Path(LE_CACHE_DIR) file_name = f"folder:{dir_path.as_posix()}.tgz" content = BytesIO() - with tar_open(file_name, mode="w:gz", fileobj=content, compresslevel=9) as tgz: + + with tar_open(file_name, mode="w:gz", fileobj=content, + compresslevel=9) as tgz: tgz.add(DATA_PATH, arcname=".") + content.seek(0, 0) - err = DB.upsert_job_cache("", file_name, content.getvalue(), job_name="certbot-renew") + err = DB.upsert_job_cache("", file_name, content.getvalue(), + job_name="certbot-renew") if err: - return jsonify({"status": "ko", "message": f"Failed to cache letsencrypt dir: {err}"}) + return jsonify({"status": "ko", + "message": f"Failed to cache letsencrypt " + f"dir: {err}"}) else: err = DB.checked_changes(["plugins"], ["letsencrypt"], True) if err: - return jsonify({"status": "ko", "message": f"Failed to cache letsencrypt dir: {err}"}) + return jsonify({"status": "ko", + "message": f"Failed to cache letsencrypt " + f"dir: {err}"}) + + if is_debug: + LOGGER.debug("Database cache updated successfully") + except Exception as e: - return jsonify({"status": "ok", "message": f"Successfully deleted certificate {cert_name}, but failed to cache letsencrypt dir: {e}"}) - return jsonify({"status": "ok", "message": f"Successfully deleted certificate {cert_name}"}) + error_msg = (f"Successfully deleted certificate {cert_name}, " + f"but failed to cache letsencrypt dir: {e}") + LOGGER.error(error_msg) + return jsonify({"status": "ok", "message": error_msg}) + + return jsonify({"status": "ok", + "message": f"Successfully deleted certificate " + f"{cert_name}"}) else: - LOGGER.error(f"Failed to delete certificate {cert_name}: {delete_proc.stdout}") - return jsonify({"status": "ko", "message": f"Failed to delete certificate {cert_name}: {delete_proc.stdout}"}) + error_msg = f"Failed to delete certificate {cert_name}: {delete_proc.stdout}" + LOGGER.error(error_msg) + return jsonify({"status": "ko", "message": error_msg}) @letsencrypt.route("/letsencrypt/") @login_required def letsencrypt_static(filename): - """ - Generalized handler for static files in the letsencrypt blueprint. - """ - return letsencrypt.send_static_file(filename) + # Serve static files for the Let's Encrypt blueprint. + # + # Args: + # filename (str): Path to the static file to serve + # + # Returns: + # Response: Flask response object for the static file + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + LOGGER.debug(f"Serving static file: {filename}") + + return letsencrypt.send_static_file(filename) \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js index 8a023a3752..df45cae585 100644 --- a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js +++ b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js @@ -1,585 +1,790 @@ (async function waitForDependencies() { - // Wait for jQuery - while (typeof jQuery === "undefined") { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - // Wait for $ to be available (in case of jQuery.noConflict()) - while (typeof $ === "undefined") { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - // Wait for DataTable to be available - while (typeof $.fn.DataTable === "undefined") { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - - $(document).ready(function () { - // Ensure i18next is loaded before using it - const t = - typeof i18next !== "undefined" - ? i18next.t - : (key, fallback) => fallback || key; // Fallback - - var actionLock = false; - const isReadOnly = $("#is-read-only").val()?.trim() === "True"; - const userReadOnly = $("#user-read-only").val()?.trim() === "True"; - - const headers = [ - { - title: "Domain", - tooltip: "Domain name for the certificate", - }, - { - title: "Common Name", - tooltip: "Common Name (CN) in the certificate", - }, - { - title: "Issuer", - tooltip: "Certificate issuing authority", - }, - { - title: "Valid From", - tooltip: "Date from which the certificate is valid", - }, - { - title: "Valid To", - tooltip: "Date until which the certificate is valid", - }, - { - title: "Preferred Profile", - tooltip: "Preferred profile for the certificate", - }, - { - title: "Challenge", - tooltip: "Challenge type used for domain validation", - }, - { - title: "Key Type", - tooltip: "Type of key used in the certificate", - }, - ]; - - // Set up the delete confirmation modal - const setupDeleteCertModal = (certs) => { - const $modalBody = $("#deleteCertContent"); - $modalBody.empty(); // Clear previous content - - if (certs.length === 1) { - $modalBody.html( - `

You are about to delete the certificate for: ${certs[0].domain}

` - ); - $("#confirmDeleteCertBtn").data("cert-name", certs[0].domain); - } else { - const certList = certs - .map((cert) => `
  • ${cert.domain}
    • `) - .join(""); - $modalBody.html( - `

    You are about to delete these certificates:

    -
      ${certList}
    ` - ); - $("#confirmDeleteCertBtn").data( - "cert-names", - certs.map((c) => c.domain) - ); - } - }; - - // Set up error modal - const showErrorModal = (title, message) => { - $("#errorModalLabel").text(title); - $("#errorModalContent").html(message); - const errorModal = new bootstrap.Modal(document.getElementById("errorModal")); - errorModal.show(); - }; - - // Handle delete button click - $("#confirmDeleteCertBtn").on("click", function () { - const certName = $(this).data("cert-name"); - const certNames = $(this).data("cert-names"); - - if (certName) { - // Delete single certificate - deleteCertificate(certName); - } else if (certNames && Array.isArray(certNames)) { - // Delete multiple certificates one by one - const deleteNext = (index) => { - if (index < certNames.length) { - deleteCertificate(certNames[index], () => { - deleteNext(index + 1); - }); - } else { - // All deleted, close modal and reload table - $("#deleteCertModal").modal("hide"); - $("#letsencrypt").DataTable().ajax.reload(); - } - }; - deleteNext(0); - } + # Wait for jQuery + while (typeof jQuery === "undefined") { + await new Promise((resolve) => setTimeout(resolve, 100)); + } - // Hide modal after starting delete process - $("#deleteCertModal").modal("hide"); - }); + # Wait for $ to be available (in case of jQuery.noConflict()) + while (typeof $ === "undefined") { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + # Wait for DataTable to be available + while (typeof $.fn.DataTable === "undefined") { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + $(document).ready(function () { + const logLevel = process.env.LOG_LEVEL; + const isDebug = logLevel === "debug"; + + if (isDebug) { + console.debug("Initializing Let's Encrypt certificate management"); + console.debug("Log level:", logLevel); + console.debug("jQuery version:", $.fn.jquery); + console.debug("DataTables version:", $.fn.DataTable.version); + } + + # Ensure i18next is loaded before using it + const t = + typeof i18next !== "undefined" + ? i18next.t + : (key, fallback) => fallback || key; + + var actionLock = false; + const isReadOnly = $("#is-read-only").val()?.trim() === "True"; + const userReadOnly = $("#user-read-only").val()?.trim() === "True"; + + if (isDebug) { + console.debug("Application state initialized:"); + console.debug("- Read-only mode:", isReadOnly); + console.debug("- User read-only:", userReadOnly); + console.debug("- Action lock:", actionLock); + console.debug("- CSRF token available:", !!$("#csrf_token").val()); + } + + const headers = [ + { + title: "Domain", + tooltip: "Domain name for the certificate", + }, + { + title: "Common Name", + tooltip: "Common Name (CN) in the certificate", + }, + { + title: "Issuer", + tooltip: "Certificate issuing authority", + }, + { + title: "Valid From", + tooltip: "Date from which the certificate is valid", + }, + { + title: "Valid To", + tooltip: "Date until which the certificate is valid", + }, + { + title: "Preferred Profile", + tooltip: "Preferred profile for the certificate", + }, + { + title: "Challenge", + tooltip: "Challenge type used for domain validation", + }, + { + title: "Key Type", + tooltip: "Type of key used in the certificate", + }, + ]; + + # Set up the delete confirmation modal for certificates + const setupDeleteCertModal = (certs) => { + if (isDebug) { + console.debug("Setting up delete modal for certificates:", + certs); + console.debug("Modal setup - certificate count:", certs.length); + } - function deleteCertificate(certName, callback) { - $.ajax({ - url: `${window.location.pathname}/delete`, - type: "POST", - contentType: "application/json", - data: JSON.stringify({ cert_name: certName }), - headers: { - "X-CSRFToken": $("#csrf_token").val(), - }, - success: function (response) { - if (response.status === "ok") { - if (callback) { - callback(); + const $modalBody = $("#deleteCertContent"); + $modalBody.empty(); + + if (certs.length === 1) { + if (isDebug) { + console.debug("Configuring modal for single certificate:", + certs[0].domain); + } + + $modalBody.html( + `

    You are about to delete the certificate for: ` + + `${certs[0].domain}

    ` + ); + $("#confirmDeleteCertBtn").data("cert-name", certs[0].domain); } else { - $("#letsencrypt").DataTable().ajax.reload(); + if (isDebug) { + console.debug("Configuring modal for multiple certificates:", + certs.map(c => c.domain)); + } + + const certList = certs + .map((cert) => `
  • ${cert.domain}
    • `) + .join(""); + $modalBody.html( + `

    You are about to delete these certificates:

    +
      ${certList}
    ` + ); + $("#confirmDeleteCertBtn").data( + "cert-names", + certs.map((c) => c.domain) + ); + } + + if (isDebug) { + console.debug("Modal configuration completed"); } - } else { - // Handle 200 OK but with error in response - showErrorModal( - "Certificate Deletion Error", - `

    Error deleting certificate ${certName}:

    ${response.message || "Unknown error"}

    ` + }; + + # Show error modal with title and message + const showErrorModal = (title, message) => { + if (isDebug) { + console.debug("Showing error modal:", title, message); + } + + $("#errorModalLabel").text(title); + $("#errorModalContent").html(message); + const errorModal = new bootstrap.Modal( + document.getElementById("errorModal") ); - if (callback) callback(); - else $("#letsencrypt").DataTable().ajax.reload(); - } - }, - error: function (xhr, status, error) { - console.error("Error deleting certificate:", error, xhr); - - // Create a more detailed error message - let errorMessage = `

    Failed to delete certificate ${certName}:

    `; - - if (xhr.responseJSON && xhr.responseJSON.message) { - errorMessage += `

    ${xhr.responseJSON.message}

    `; - } else if (xhr.responseText) { - try { - const parsedError = JSON.parse(xhr.responseText); - errorMessage += `

    ${parsedError.message || error}

    `; - } catch (e) { - // If can't parse JSON, use the raw response text if not too large - if (xhr.responseText.length < 200) { - errorMessage += `

    ${xhr.responseText}

    `; - } else { - errorMessage += `

    ${error || "Unknown error"}

    `; - } + errorModal.show(); + }; + + # Handle delete button click events + $("#confirmDeleteCertBtn").on("click", function () { + const certName = $(this).data("cert-name"); + const certNames = $(this).data("cert-names"); + + if (isDebug) { + console.debug("Delete button clicked:", + { certName, certNames }); } - } else { - errorMessage += `

    ${error || "Unknown error"}

    `; - } - showErrorModal("Certificate Deletion Failed", errorMessage); + if (certName) { + deleteCertificate(certName); + } else if (certNames && Array.isArray(certNames)) { + # Delete multiple certificates sequentially + const deleteNext = (index) => { + if (index < certNames.length) { + deleteCertificate(certNames[index], () => { + deleteNext(index + 1); + }); + } else { + $("#deleteCertModal").modal("hide"); + $("#letsencrypt").DataTable().ajax.reload(); + } + }; + deleteNext(0); + } - if (callback) callback(); - else $("#letsencrypt").DataTable().ajax.reload(); - }, - }); - } + $("#deleteCertModal").modal("hide"); + }); + + # Delete a single certificate with optional callback + function deleteCertificate(certName, callback) { + if (isDebug) { + console.debug("Starting certificate deletion process:"); + console.debug("- Certificate name:", certName); + console.debug("- Has callback:", !!callback); + console.debug("- Request URL:", + `${window.location.pathname}/delete`); + } - // DataTable Layout and Buttons - const layout = { - top1: { - searchPanes: { - viewTotal: true, - cascadePanes: true, - collapse: false, - columns: [2, 5, 6, 7], // Issuer, Preferred Profile, Challenge and Key Type - }, - }, - topStart: {}, - topEnd: { - search: true, - buttons: [ - { - extend: "auto_refresh", - className: - "btn btn-sm btn-outline-primary d-flex align-items-center", - }, - { - extend: "toggle_filters", - className: "btn btn-sm btn-outline-primary toggle-filters", - }, - ], - }, - bottomStart: { - pageLength: { - menu: [10, 25, 50, 100, { label: "All", value: -1 }], - }, - info: true, - }, - }; - - layout.topStart.buttons = [ - { - extend: "colvis", - columns: "th:not(:nth-child(-n+3)):not(:last-child)", - text: `${t( - "button.columns", - "Columns" - )}`, - className: "btn btn-sm btn-outline-primary rounded-start", - columnText: function (dt, idx, title) { - return `${idx + 1}. ${title}`; - }, - }, - { - extend: "colvisRestore", - text: `${t( - "button.reset_columns", - "Reset columns" - )}`, - className: "btn btn-sm btn-outline-primary d-none d-md-inline", - }, - { - extend: "collection", - text: `${t( - "button.export", - "Export" - )}`, - className: "btn btn-sm btn-outline-primary", - buttons: [ - { - extend: "copy", - text: `${t( - "button.copy_visible", - "Copy visible" - )}`, - exportOptions: { - columns: ":visible:not(:nth-child(-n+2)):not(:last-child)", + const requestData = { cert_name: certName }; + const csrfToken = $("#csrf_token").val(); + + if (isDebug) { + console.debug("Request payload:", requestData); + console.debug("CSRF token:", csrfToken ? "present" : "missing"); + } + + $.ajax({ + url: `${window.location.pathname}/delete`, + type: "POST", + contentType: "application/json", + data: JSON.stringify(requestData), + headers: { + "X-CSRFToken": csrfToken, + }, + beforeSend: function(xhr) { + if (isDebug) { + console.debug("AJAX request starting for:", certName); + console.debug("Request headers:", xhr.getAllResponseHeaders()); + } + }, + success: function (response) { + if (isDebug) { + console.debug("Delete response received:"); + console.debug("- Status:", response.status); + console.debug("- Message:", response.message); + console.debug("- Full response:", response); + } + + if (response.status === "ok") { + if (isDebug) { + console.debug("Certificate deletion successful:", + certName); + } + + if (callback) { + if (isDebug) { + console.debug("Executing callback function"); + } + callback(); + } else { + if (isDebug) { + console.debug("Reloading DataTable data"); + } + $("#letsencrypt").DataTable().ajax.reload(); + } + } else { + if (isDebug) { + console.debug("Certificate deletion failed:", + response.message); + } + + showErrorModal( + "Certificate Deletion Error", + `

    Error deleting certificate ` + + `${certName}:

    ` + + `

    ${response.message || "Unknown error"}

    ` + ); + if (callback) callback(); + else $("#letsencrypt").DataTable().ajax.reload(); + } + }, + error: function (xhr, status, error) { + if (isDebug) { + console.debug("AJAX error details:"); + console.debug("- XHR status:", xhr.status); + console.debug("- Status text:", status); + console.debug("- Error:", error); + console.debug("- Response text:", xhr.responseText); + console.debug("- Response JSON:", xhr.responseJSON); + } + + console.error("Error deleting certificate:", error, xhr); + + let errorMessage = `

    Failed to delete certificate ` + + `${certName}:

    `; + + if (xhr.responseJSON && xhr.responseJSON.message) { + errorMessage += `

    ${xhr.responseJSON.message}

    `; + } else if (xhr.responseText) { + try { + const parsedError = JSON.parse(xhr.responseText); + errorMessage += + `

    ${parsedError.message || error}

    `; + } catch (e) { + if (isDebug) { + console.debug("Failed to parse error response:", + e); + } + if (xhr.responseText.length < 200) { + errorMessage += `

    ${xhr.responseText}

    `; + } else { + errorMessage += `

    ${error || + "Unknown error"}

    `; + } + } + } else { + errorMessage += `

    ${error || "Unknown error"}

    `; + } + + showErrorModal("Certificate Deletion Failed", + errorMessage); + + if (callback) callback(); + else $("#letsencrypt").DataTable().ajax.reload(); + }, + complete: function(xhr, status) { + if (isDebug) { + console.debug("AJAX request completed:"); + console.debug("- Final status:", status); + console.debug("- Certificate:", certName); + } + } + }); + } + + # DataTable Layout and Button configuration + const layout = { + top1: { + searchPanes: { + viewTotal: true, + cascadePanes: true, + collapse: false, + # Issuer, Preferred Profile, Challenge and Key Type + columns: [2, 5, 6, 7], + }, }, - }, - { - extend: "csv", - text: `CSV`, - bom: true, - filename: "bw_certificates", - exportOptions: { - modifier: { search: "none" }, - columns: ":not(:nth-child(-n+2)):not(:last-child)", + topStart: {}, + topEnd: { + search: true, + buttons: [ + { + extend: "auto_refresh", + className: ( + "btn btn-sm btn-outline-primary " + + "d-flex align-items-center" + ), + }, + { + extend: "toggle_filters", + className: "btn btn-sm btn-outline-primary " + + "toggle-filters", + }, + ], }, - }, - { - extend: "excel", - text: `Excel`, - filename: "bw_certificates", - exportOptions: { - modifier: { search: "none" }, - columns: ":not(:nth-child(-n+2)):not(:last-child)", + bottomStart: { + pageLength: { + menu: [10, 25, 50, 100, + { label: "All", value: -1 }], + }, + info: true, }, - }, - ], - }, - { - extend: "collection", - text: `${t( - "button.actions", - "Actions" - )}`, - className: "btn btn-sm btn-outline-primary action-button disabled", - buttons: [{ extend: "delete_cert", className: "text-danger" }], - }, - ]; - - let autoRefresh = false; - let autoRefreshInterval = null; - const sessionAutoRefresh = sessionStorage.getItem("letsencryptAutoRefresh"); - - function toggleAutoRefresh() { - autoRefresh = !autoRefresh; - sessionStorage.setItem("letsencryptAutoRefresh", autoRefresh); - if (autoRefresh) { - $(".bx-loader") - .addClass("bx-spin") - .closest(".btn") - .removeClass("btn-outline-primary") - .addClass("btn-primary"); - if (autoRefreshInterval) clearInterval(autoRefreshInterval); - autoRefreshInterval = setInterval(() => { - if (!autoRefresh) { - clearInterval(autoRefreshInterval); - autoRefreshInterval = null; - } else { - $("#letsencrypt").DataTable().ajax.reload(null, false); - } - }, 10000); // 10 seconds - } else { - $(".bx-loader") - .removeClass("bx-spin") - .closest(".btn") - .removeClass("btn-primary") - .addClass("btn-outline-primary"); - if (autoRefreshInterval) { - clearInterval(autoRefreshInterval); - autoRefreshInterval = null; + }; + + if (isDebug) { + console.debug("DataTable layout configuration:"); + console.debug("- Search panes columns:", layout.top1.searchPanes.columns); + console.debug("- Page length options:", layout.bottomStart.pageLength.menu); + console.debug("- Layout structure:", layout); } - } - } - if (sessionAutoRefresh === "true") { - toggleAutoRefresh(); - } + layout.topStart.buttons = [ + { + extend: "colvis", + columns: "th:not(:nth-child(-n+3)):not(:last-child)", + text: ( + `${t( + "button.columns", + "Columns" + )}` + ), + className: "btn btn-sm btn-outline-primary rounded-start", + columnText: function (dt, idx, title) { + return `${idx + 1}. ${title}`; + }, + }, + { + extend: "colvisRestore", + text: ( + `${t( + "button.reset_columns", + "Reset columns" + )}` + ), + className: "btn btn-sm btn-outline-primary d-none d-md-inline", + }, + { + extend: "collection", + text: ( + `${t( + "button.export", + "Export" + )}` + ), + className: "btn btn-sm btn-outline-primary", + buttons: [ + { + extend: "copy", + text: ( + `${t( + "button.copy_visible", + "Copy visible" + )}` + ), + exportOptions: { + columns: ( + ":visible:not(:nth-child(-n+2)):" + + "not(:last-child)" + ), + }, + }, + { + extend: "csv", + text: ( + `CSV` + ), + bom: true, + filename: "bw_certificates", + exportOptions: { + modifier: { search: "none" }, + columns: ( + ":not(:nth-child(-n+2)):not(:last-child)" + ), + }, + }, + { + extend: "excel", + text: ( + `Excel` + ), + filename: "bw_certificates", + exportOptions: { + modifier: { search: "none" }, + columns: ( + ":not(:nth-child(-n+2)):not(:last-child)" + ), + }, + }, + ], + }, + { + extend: "collection", + text: ( + `${t( + "button.actions", + "Actions" + )}` + ), + className: ( + "btn btn-sm btn-outline-primary action-button disabled" + ), + buttons: [ + { extend: "delete_cert", className: "text-danger" } + ], + }, + ]; - const getSelectedCertificates = () => { - const certs = []; - $("tr.selected").each(function () { - const $row = $(this); - const domain = $row.find("td:eq(2)").text().trim(); - certs.push({ - domain: domain, - }); - }); - return certs; - }; - - $.fn.dataTable.ext.buttons.auto_refresh = { - text: '  Auto refresh', - action: (e, dt, node, config) => { - toggleAutoRefresh(); - }, - }; - - $.fn.dataTable.ext.buttons.delete_cert = { - text: `Delete certificate`, - action: function (e, dt, node, config) { - if (isReadOnly) { - alert( - t( - "alert.readonly_mode", - "This action is not allowed in read-only mode." - ) - ); - return; + let autoRefresh = false; + let autoRefreshInterval = null; + const sessionAutoRefresh = + sessionStorage.getItem("letsencryptAutoRefresh"); + + # Toggle auto-refresh functionality + function toggleAutoRefresh() { + autoRefresh = !autoRefresh; + sessionStorage.setItem("letsencryptAutoRefresh", autoRefresh); + + if (isDebug) { + console.debug("Auto-refresh toggled:", autoRefresh); + } + + if (autoRefresh) { + $(".bx-loader") + .addClass("bx-spin") + .closest(".btn") + .removeClass("btn-outline-primary") + .addClass("btn-primary"); + + if (autoRefreshInterval) clearInterval(autoRefreshInterval); + + autoRefreshInterval = setInterval(() => { + if (!autoRefresh) { + clearInterval(autoRefreshInterval); + autoRefreshInterval = null; + } else { + $("#letsencrypt").DataTable().ajax.reload(null, false); + } + }, 10000); + } else { + $(".bx-loader") + .removeClass("bx-spin") + .closest(".btn") + .removeClass("btn-primary") + .addClass("btn-outline-primary"); + + if (autoRefreshInterval) { + clearInterval(autoRefreshInterval); + autoRefreshInterval = null; + } + } } - if (actionLock) return; - actionLock = true; - $(".dt-button-background").click(); - - const certs = getSelectedCertificates(); - if (certs.length === 0) { - actionLock = false; - return; + + if (sessionAutoRefresh === "true") { + toggleAutoRefresh(); } - setupDeleteCertModal(certs); - - // Show the modal - const deleteModal = new bootstrap.Modal( - document.getElementById("deleteCertModal") - ); - deleteModal.show(); - - actionLock = false; - }, - }; - - // Create columns configuration - function buildColumnDefs() { - return [ - { orderable: false, className: "dtr-control", targets: 0 }, - { orderable: false, render: DataTable.render.select(), targets: 1 }, - { type: "string", targets: 2 }, // domain - { orderable: true, targets: -1 }, - { - targets: [5, 6], - render: function (data, type, row) { - if (type === "display" || type === "filter") { - const date = new Date(data); - if (!isNaN(date.getTime())) { - return date.toLocaleString(); - } + + # Get currently selected certificates from DataTable + const getSelectedCertificates = () => { + const certs = []; + $("tr.selected").each(function () { + const $row = $(this); + const domain = $row.find("td:eq(2)").text().trim(); + certs.push({ domain: domain }); + }); + + if (isDebug) { + console.debug("Selected certificates:", certs); } - return data; - }, - }, - { - searchPanes: { - show: true, - combiner: "or", - header: t("searchpane.issuer", "Issuer"), - }, - targets: 2, // Issuer column - }, - { - searchPanes: { - show: true, - header: t("searchpane.preferred_profile", "Preferred Profile"), - combiner: "or", - }, - targets: 5, // Preferred Profile column - }, - { - searchPanes: { - show: true, - header: t("searchpane.challenge", "Challenge"), - combiner: "or", - }, - targets: 6, // Challenge column - }, - { - searchPanes: { - show: true, - header: t("searchpane.key_type", "Key Type"), - combiner: "or", - }, - targets: 7, // Key Type column - }, - ]; - } - // Define the columns for the DataTable - function buildColumns() { - return [ - { - data: null, - defaultContent: "", - orderable: false, - className: "dtr-control", - }, - { data: null, defaultContent: "", orderable: false }, - { - data: "domain", - title: "Domain", - }, - { - data: "common_name", - title: "Common Name", - }, - { - data: "issuer", - title: "Issuer", - }, - { - data: "valid_from", - title: "Valid From", - }, - { - data: "valid_to", - title: "Valid To", - }, - { - data: "preferred_profile", - title: "Preferred Profile", - }, - { - data: "challenge", - title: "Challenge", - }, - { - data: "key_type", - title: "Key Type", - }, - { - data: "serial_number", - title: "Serial Number", - }, - { - data: "fingerprint", - title: "Fingerprint", - }, - { - data: "version", - title: "Version", - }, - ]; - } + return certs; + }; - // Utility function to manage header tooltips - function updateHeaderTooltips(selector, headers) { - $(selector) - .find("th") - .each((index, element) => { - const $th = $(element); - const tooltip = headers[index] ? headers[index].tooltip : ""; - if (!tooltip) return; - - $th.attr({ - "data-bs-toggle": "tooltip", - "data-bs-placement": "bottom", - title: tooltip, - }); - }); + # Custom DataTable button for auto-refresh + $.fn.dataTable.ext.buttons.auto_refresh = { + text: ( + '' + + '  ' + + 'Auto refresh' + ), + action: (e, dt, node, config) => { + toggleAutoRefresh(); + }, + }; - $('[data-bs-toggle="tooltip"]').tooltip("dispose").tooltip(); - } + # Custom DataTable button for certificate deletion + $.fn.dataTable.ext.buttons.delete_cert = { + text: ( + `` + + `Delete certificate` + ), + action: function (e, dt, node, config) { + if (isReadOnly) { + alert( + t( + "alert.readonly_mode", + "This action is not allowed in read-only mode." + ) + ); + return; + } + + if (actionLock) return; + actionLock = true; + $(".dt-button-background").click(); + + const certs = getSelectedCertificates(); + if (certs.length === 0) { + actionLock = false; + return; + } + + setupDeleteCertModal(certs); + + const deleteModal = new bootstrap.Modal( + document.getElementById("deleteCertModal") + ); + deleteModal.show(); + + actionLock = false; + }, + }; - // Initialize the DataTable with columns and configuration - const letsencrypt_config = { - tableSelector: "#letsencrypt", - tableName: "letsencrypt", - columnVisibilityCondition: (column) => column > 2 && column < 13, - dataTableOptions: { - columnDefs: buildColumnDefs(), - order: [[2, "asc"]], // Sort by domain name - autoFill: false, - responsive: true, - select: { - style: "multi+shift", - selector: "td:nth-child(2)", - headerCheckbox: true, - }, - layout: layout, - processing: true, - serverSide: true, - ajax: { - url: `${window.location.pathname}/fetch`, - type: "POST", - data: function (d) { - d.csrf_token = $("#csrf_token").val(); - return d; - }, - // Add error handling for ajax requests - error: function (jqXHR, textStatus, errorThrown) { - console.error("DataTables AJAX error:", textStatus, errorThrown); - $("#letsencrypt").addClass("d-none"); - $("#letsencrypt-waiting") - .removeClass("d-none") - .text( - "Error loading certificates. Please try refreshing the page." - ) - .addClass("text-danger"); - // Remove any loading indicators - $(".dataTables_processing").hide(); - }, - }, - columns: buildColumns(), - initComplete: function (settings, json) { - $("#letsencrypt_wrapper .btn-secondary").removeClass("btn-secondary"); - - // Hide loading message and show table - $("#letsencrypt-waiting").addClass("d-none"); - $("#letsencrypt").removeClass("d-none"); - - if (isReadOnly) { - const titleKey = userReadOnly - ? "tooltip.readonly_user_action_disabled" - : "tooltip.readonly_db_action_disabled"; - const defaultTitle = userReadOnly - ? "Your account is readonly, action disabled." - : "The database is in readonly, action disabled."; - } - }, - headerCallback: function (thead) { - updateHeaderTooltips(thead, headers); - }, - }, - }; - - const dt = initializeDataTable(letsencrypt_config); - dt.on("draw.dt", function () { - updateHeaderTooltips(dt.table().header(), headers); - $(".tooltip").remove(); - }); - dt.on("column-visibility.dt", function (e, settings, column, state) { - updateHeaderTooltips(dt.table().header(), headers); - $(".tooltip").remove(); - }); + # Build column definitions for DataTable + function buildColumnDefs() { + return [ + { + orderable: false, + className: "dtr-control", + targets: 0 + }, + { + orderable: false, + render: DataTable.render.select(), + targets: 1 + }, + { type: "string", targets: 2 }, + { orderable: true, targets: -1 }, + { + targets: [5, 6], + render: function (data, type, row) { + if (type === "display" || type === "filter") { + const date = new Date(data); + if (!isNaN(date.getTime())) { + return date.toLocaleString(); + } + } + return data; + }, + }, + { + searchPanes: { + show: true, + combiner: "or", + header: t("searchpane.issuer", "Issuer"), + }, + targets: 2, + }, + { + searchPanes: { + show: true, + header: t("searchpane.preferred_profile", + "Preferred Profile"), + combiner: "or", + }, + targets: 5, + }, + { + searchPanes: { + show: true, + header: t("searchpane.challenge", "Challenge"), + combiner: "or", + }, + targets: 6, + }, + { + searchPanes: { + show: true, + header: t("searchpane.key_type", "Key Type"), + combiner: "or", + }, + targets: 7, + }, + ]; + } + + # Define the columns for the DataTable + function buildColumns() { + return [ + { + data: null, + defaultContent: "", + orderable: false, + className: "dtr-control", + }, + { data: null, defaultContent: "", orderable: false }, + { data: "domain", title: "Domain" }, + { data: "common_name", title: "Common Name" }, + { data: "issuer", title: "Issuer" }, + { data: "valid_from", title: "Valid From" }, + { data: "valid_to", title: "Valid To" }, + { data: "preferred_profile", title: "Preferred Profile" }, + { data: "challenge", title: "Challenge" }, + { data: "key_type", title: "Key Type" }, + { data: "serial_number", title: "Serial Number" }, + { data: "fingerprint", title: "Fingerprint" }, + { data: "version", title: "Version" }, + ]; + } + + # Manage header tooltips for DataTable columns + function updateHeaderTooltips(selector, headers) { + $(selector) + .find("th") + .each((index, element) => { + const $th = $(element); + const tooltip = headers[index] ? + headers[index].tooltip : ""; + if (!tooltip) return; + + $th.attr({ + "data-bs-toggle": "tooltip", + "data-bs-placement": "bottom", + title: tooltip, + }); + }); + + $('[data-bs-toggle="tooltip"]').tooltip("dispose").tooltip(); + } + + # Initialize the DataTable with complete configuration + const letsencrypt_config = { + tableSelector: "#letsencrypt", + tableName: "letsencrypt", + columnVisibilityCondition: (column) => column > 2 && column < 13, + dataTableOptions: { + columnDefs: buildColumnDefs(), + order: [[2, "asc"]], + autoFill: false, + responsive: true, + select: { + style: "multi+shift", + selector: "td:nth-child(2)", + headerCheckbox: true, + }, + layout: layout, + processing: true, + serverSide: true, + ajax: { + url: `${window.location.pathname}/fetch`, + type: "POST", + data: function (d) { + if (isDebug) { + console.debug("DataTable AJAX request data:", d); + console.debug("Request parameters:"); + console.debug("- Draw:", d.draw); + console.debug("- Start:", d.start); + console.debug("- Length:", d.length); + console.debug("- Search value:", d.search?.value); + } + + d.csrf_token = $("#csrf_token").val(); + return d; + }, + error: function (jqXHR, textStatus, errorThrown) { + if (isDebug) { + console.debug("DataTable AJAX error details:"); + console.debug("- Status:", jqXHR.status); + console.debug("- Status text:", textStatus); + console.debug("- Error:", errorThrown); + console.debug("- Response text:", jqXHR.responseText); + console.debug("- Response headers:", + jqXHR.getAllResponseHeaders()); + } + + console.error("DataTables AJAX error:", + textStatus, errorThrown); + + $("#letsencrypt").addClass("d-none"); + $("#letsencrypt-waiting") + .removeClass("d-none") + .text("Error loading certificates. " + + "Please try refreshing the page.") + .addClass("text-danger"); + + $(".dataTables_processing").hide(); + }, + success: function(data, textStatus, jqXHR) { + if (isDebug) { + console.debug("DataTable AJAX success:"); + console.debug("- Records total:", data.recordsTotal); + console.debug("- Records filtered:", data.recordsFiltered); + console.debug("- Data length:", data.data?.length); + console.debug("- Draw number:", data.draw); + } + } + }, + columns: buildColumns(), + initComplete: function (settings, json) { + if (isDebug) { + console.debug("DataTable initialized with settings:", + settings); + } + + $("#letsencrypt_wrapper .btn-secondary") + .removeClass("btn-secondary"); + + $("#letsencrypt-waiting").addClass("d-none"); + $("#letsencrypt").removeClass("d-none"); + + if (isReadOnly) { + const titleKey = userReadOnly + ? "tooltip.readonly_user_action_disabled" + : "tooltip.readonly_db_action_disabled"; + const defaultTitle = userReadOnly + ? "Your account is readonly, action disabled." + : "The database is in readonly, action disabled."; + } + }, + headerCallback: function (thead) { + updateHeaderTooltips(thead, headers); + }, + }, + }; - // Add selection event handler for toggle action button - dt.on("select.dt deselect.dt", function () { - const count = dt.rows({ selected: true }).count(); - $(".action-button").toggleClass("disabled", count === 0); + const dt = initializeDataTable(letsencrypt_config); + + dt.on("draw.dt", function () { + updateHeaderTooltips(dt.table().header(), headers); + $(".tooltip").remove(); + }); + + dt.on("column-visibility.dt", function (e, settings, column, state) { + updateHeaderTooltips(dt.table().header(), headers); + $(".tooltip").remove(); + }); + + # Toggle action button based on selection + dt.on("select.dt deselect.dt", function () { + const count = dt.rows({ selected: true }).count(); + $(".action-button").toggleClass("disabled", count === 0); + + if (isDebug) { + console.debug("Selection changed, count:", count); + } + }); }); - }); -})(); +})(); \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/hooks.py b/src/common/core/letsencrypt/ui/hooks.py index 0ca87d78b6..518406e9c6 100644 --- a/src/common/core/letsencrypt/ui/hooks.py +++ b/src/common/core/letsencrypt/ui/hooks.py @@ -1,30 +1,74 @@ +from logging import getLogger +from os import getenv from flask import request -# Default column visibility settings for letsencrypt tables +# Default column visibility settings for Let's Encrypt certificate tables +# Key represents column index, value indicates if column is visible by default COLUMNS_PREFERENCES_DEFAULTS = { - "3": True, - "4": True, - "5": True, - "6": True, - "7": True, - "8": True, - "9": False, - "10": False, - "11": True, + "3": True, # Common Name + "4": True, # Issuer + "5": True, # Valid From + "6": True, # Valid To + "7": True, # Preferred Profile + "8": True, # Challenge + "9": False, # Serial Number (hidden by default) + "10": False, # Fingerprint (hidden by default) + "11": True, # Key Type } def context_processor(): - """ - Flask context processor to inject variables into templates. - - This adds: - - Column preference defaults for tables - - Extra pages visibility based on user permissions - """ - if request.path.startswith(("/check", "/setup", "/loading", "/login", "/totp", "/logout")): + # Flask context processor to inject variables into templates. + # + # Provides template context data for the Let's Encrypt certificate + # management interface. Injects column preferences and other UI + # configuration data that templates need for proper rendering. + # + # Returns: + # dict: Dictionary containing template context variables including + # column preferences for DataTables and page visibility settings. + # Returns None for excluded paths that don't need context injection. + logger = getLogger("UI") + is_debug = getenv("LOG_LEVEL") == "debug" + + if is_debug: + logger.debug("Context processor called") + logger.debug(f"Request path: {request.path}") + logger.debug(f"Request method: {request.method}") + logger.debug(f"Request endpoint: {getattr(request, 'endpoint', 'unknown')}") + + # Skip context processing for system/auth pages that don't need it + excluded_paths = [ + "/check", "/setup", "/loading", + "/login", "/totp", "/logout" + ] + + # Check if current path should be excluded + path_excluded = request.path.startswith(tuple(excluded_paths)) + + if path_excluded: + if is_debug: + logger.debug(f"Path {request.path} is excluded from context processing") + for excluded_path in excluded_paths: + if request.path.startswith(excluded_path): + logger.debug(f" Matched exclusion pattern: {excluded_path}") + break return None - data = {"columns_preferences_defaults_letsencrypt": COLUMNS_PREFERENCES_DEFAULTS} + if is_debug: + logger.debug(f"Processing context for path: {request.path}") + logger.debug(f"Column preferences to inject:") + for col_id, visible in COLUMNS_PREFERENCES_DEFAULTS.items(): + logger.debug(f" Column {col_id}: {'visible' if visible else 'hidden'}") + + # Prepare context data for templates + data = { + "columns_preferences_defaults_letsencrypt": COLUMNS_PREFERENCES_DEFAULTS + } + + if is_debug: + logger.debug(f"Context processor returning {len(data)} variables") + logger.debug(f"Context data keys: {list(data.keys())}") + logger.debug(f"Let's Encrypt preferences: {len(COLUMNS_PREFERENCES_DEFAULTS)} columns configured") - return data + return data \ No newline at end of file From 1b5b34a71758f5dc468e1ce78bffed775cf03798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 09:25:38 +0200 Subject: [PATCH 06/15] Add field "OCSP" to ui - show if the certificate supports OCSP --- src/common/core/letsencrypt/ui/actions.py | 45 ++++++++++++++++++- .../letsencrypt/ui/blueprints/letsencrypt.py | 42 +++++++++++++++++ .../ui/blueprints/static/js/main.js | 20 +++++++-- src/common/core/letsencrypt/ui/hooks.py | 17 +++++-- 4 files changed, 116 insertions(+), 8 deletions(-) diff --git a/src/common/core/letsencrypt/ui/actions.py b/src/common/core/letsencrypt/ui/actions.py index e8668caa16..478a98d17d 100644 --- a/src/common/core/letsencrypt/ui/actions.py +++ b/src/common/core/letsencrypt/ui/actions.py @@ -10,6 +10,7 @@ from uuid import uuid4 from cryptography import x509 +from cryptography.x509 import oid from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes @@ -153,6 +154,7 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: "challenge": [], "authenticator": [], "key_type": [], + "ocsp_support": [], } total_certs_processed = 0 @@ -191,6 +193,7 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: "challenge": "Unknown", "authenticator": "Unknown", "key_type": "Unknown", + "ocsp_support": "Unknown", } # Parse the certificate file @@ -269,12 +272,42 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Extract version cert_info["version"] = cert.version.name + # Check for OCSP support via Authority Information Access extension + try: + aia_ext = cert.extensions.get_extension_for_oid( + oid.ExtensionOID.AUTHORITY_INFORMATION_ACCESS + ) + ocsp_urls = [] + for access_description in aia_ext.value: + if (access_description.access_method == + oid.AuthorityInformationAccessOID.OCSP): + ocsp_urls.append(str(access_description.access_location.value)) + + if ocsp_urls: + cert_info["ocsp_support"] = "Yes" + if is_debug: + logger.debug(f"OCSP URLs found for {domain}: {ocsp_urls}") + else: + cert_info["ocsp_support"] = "No" + if is_debug: + logger.debug(f"AIA extension found for {domain} but no OCSP URLs") + + except x509.ExtensionNotFound: + cert_info["ocsp_support"] = "No" + if is_debug: + logger.debug(f"No Authority Information Access extension found for {domain}") + except Exception as ocsp_error: + cert_info["ocsp_support"] = "Unknown" + if is_debug: + logger.debug(f"Error checking OCSP support for {domain}: {ocsp_error}") + if is_debug: logger.debug(f"Certificate processing completed for {domain}") logger.debug(f" - Serial: {cert_info['serial_number']}") logger.debug(f" - Version: {cert_info['version']}") logger.debug(f" - Subject: {cert_info['common_name']}") logger.debug(f" - Issuer: {cert_info['issuer']}") + logger.debug(f" - OCSP Support: {cert_info['ocsp_support']}") except BaseException as e: error_msg = f"Error while parsing certificate {cert_file}: {e}" @@ -349,6 +382,12 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: logger.debug(f"Certificate retrieval complete. Processed " f"{total_certs_processed} certificates from " f"{len(folder_paths)} folders") + + # Summary of OCSP support + ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} + for ocsp_status in certificates.get('ocsp_support', []): + ocsp_support_counts[ocsp_status] = ocsp_support_counts.get(ocsp_status, 0) + 1 + logger.debug(f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -393,6 +432,7 @@ def pre_render(app, *args, **kwargs): "challenge": [], "authenticator": [], "key_type": [], + "ocsp_support": [], }, "order": { "column": 5, @@ -461,7 +501,10 @@ def pre_render(app, *args, **kwargs): logger.debug("Sample certificate data (first certificate):") for key in cert_data: value = cert_data[key][0] if cert_data[key] else "None" - logger.debug(f" {key}: {value}") + if key == "ocsp_support": + logger.debug(f" {key}: {value} (OCSP support detected)") + else: + logger.debug(f" {key}: {value}") ret["list_certificates"]["data"] = cert_data diff --git a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py index e42777b363..a22ba86579 100644 --- a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py +++ b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py @@ -8,6 +8,7 @@ from traceback import format_exc from cryptography import x509 +from cryptography.x509 import oid from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from flask import Blueprint, render_template, request, jsonify @@ -149,6 +150,7 @@ def retrieve_certificates(): "challenge": [], "authenticator": [], "key_type": [], + "ocsp_support": [], } cert_files = list(Path(DATA_PATH).joinpath("live").glob("*/fullchain.pem")) @@ -178,6 +180,7 @@ def retrieve_certificates(): "challenge": "Unknown", "authenticator": "Unknown", "key_type": "Unknown", + "ocsp_support": "Unknown", } try: @@ -235,11 +238,41 @@ def retrieve_certificates(): cert_info["fingerprint"] = cert.fingerprint(hashes.SHA256()).hex() cert_info["version"] = cert.version.name + # Check for OCSP support via Authority Information Access extension + try: + aia_ext = cert.extensions.get_extension_for_oid( + oid.ExtensionOID.AUTHORITY_INFORMATION_ACCESS + ) + ocsp_urls = [] + for access_description in aia_ext.value: + if (access_description.access_method == + oid.AuthorityInformationAccessOID.OCSP): + ocsp_urls.append(str(access_description.access_location.value)) + + if ocsp_urls: + cert_info["ocsp_support"] = "Yes" + if is_debug: + LOGGER.debug(f"OCSP URLs found: {ocsp_urls}") + else: + cert_info["ocsp_support"] = "No" + if is_debug: + LOGGER.debug("AIA extension found but no OCSP URLs") + + except x509.ExtensionNotFound: + cert_info["ocsp_support"] = "No" + if is_debug: + LOGGER.debug("No Authority Information Access extension found") + except Exception as ocsp_error: + cert_info["ocsp_support"] = "Unknown" + if is_debug: + LOGGER.debug(f"Error checking OCSP support: {ocsp_error}") + if is_debug: LOGGER.debug(f"Certificate details extracted:") LOGGER.debug(f" - Serial: {cert_info['serial_number']}") LOGGER.debug(f" - Fingerprint: {cert_info['fingerprint'][:16]}...") LOGGER.debug(f" - Version: {cert_info['version']}") + LOGGER.debug(f" - OCSP Support: {cert_info['ocsp_support']}") except BaseException as e: LOGGER.debug(format_exc()) @@ -321,6 +354,11 @@ def retrieve_certificates(): if is_debug: LOGGER.debug(f"Retrieved {len(certificates['domain'])} certificates") + # Summary of OCSP support + ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} + for ocsp_status in certificates.get('ocsp_support', []): + ocsp_support_counts[ocsp_status] = ocsp_support_counts.get(ocsp_status, 0) + 1 + LOGGER.debug(f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -380,11 +418,15 @@ def letsencrypt_fetch(): "challenge": certs.get("challenge", [""])[i], "authenticator": certs.get("authenticator", [""])[i], "key_type": certs.get("key_type", [""])[i], + "ocsp_support": certs.get("ocsp_support", [""])[i], } cert_list.append(cert_data) if is_debug: LOGGER.debug(f"Added certificate to list: {domain}") + LOGGER.debug(f" - OCSP Support: {cert_data['ocsp_support']}") + LOGGER.debug(f" - Challenge: {cert_data['challenge']}") + LOGGER.debug(f" - Key Type: {cert_data['key_type']}") except BaseException as e: LOGGER.debug(format_exc()) diff --git a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js index df45cae585..6e3112e7a6 100644 --- a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js +++ b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js @@ -76,6 +76,10 @@ title: "Key Type", tooltip: "Type of key used in the certificate", }, + { + title: "OCSP", + tooltip: "Online Certificate Status Protocol support", + }, ]; # Set up the delete confirmation modal for certificates @@ -302,8 +306,8 @@ viewTotal: true, cascadePanes: true, collapse: false, - # Issuer, Preferred Profile, Challenge and Key Type - columns: [2, 5, 6, 7], + # Issuer, Preferred Profile, Challenge, Key Type, and OCSP + columns: [2, 5, 6, 7, 8], }, }, topStart: {}, @@ -338,6 +342,7 @@ console.debug("- Search panes columns:", layout.top1.searchPanes.columns); console.debug("- Page length options:", layout.bottomStart.pageLength.menu); console.debug("- Layout structure:", layout); + console.debug("- Headers count:", headers.length); } layout.topStart.buttons = [ @@ -623,6 +628,14 @@ }, targets: 7, }, + { + searchPanes: { + show: true, + header: t("searchpane.ocsp", "OCSP Support"), + combiner: "or", + }, + targets: 8, + }, ]; } @@ -644,6 +657,7 @@ { data: "preferred_profile", title: "Preferred Profile" }, { data: "challenge", title: "Challenge" }, { data: "key_type", title: "Key Type" }, + { data: "ocsp_support", title: "OCSP" }, { data: "serial_number", title: "Serial Number" }, { data: "fingerprint", title: "Fingerprint" }, { data: "version", title: "Version" }, @@ -674,7 +688,7 @@ const letsencrypt_config = { tableSelector: "#letsencrypt", tableName: "letsencrypt", - columnVisibilityCondition: (column) => column > 2 && column < 13, + columnVisibilityCondition: (column) => column > 2 && column < 14, dataTableOptions: { columnDefs: buildColumnDefs(), order: [[2, "asc"]], diff --git a/src/common/core/letsencrypt/ui/hooks.py b/src/common/core/letsencrypt/ui/hooks.py index 518406e9c6..f5d7337dfc 100644 --- a/src/common/core/letsencrypt/ui/hooks.py +++ b/src/common/core/letsencrypt/ui/hooks.py @@ -11,9 +11,11 @@ "6": True, # Valid To "7": True, # Preferred Profile "8": True, # Challenge - "9": False, # Serial Number (hidden by default) - "10": False, # Fingerprint (hidden by default) - "11": True, # Key Type + "9": True, # Key Type + "10": True, # OCSP Support + "11": False, # Serial Number (hidden by default) + "12": False, # Fingerprint (hidden by default) + "13": True, # Version } @@ -58,8 +60,15 @@ def context_processor(): if is_debug: logger.debug(f"Processing context for path: {request.path}") logger.debug(f"Column preferences to inject:") + column_names = { + "3": "Common Name", "4": "Issuer", "5": "Valid From", + "6": "Valid To", "7": "Preferred Profile", "8": "Challenge", + "9": "Key Type", "10": "OCSP Support", "11": "Serial Number", + "12": "Fingerprint", "13": "Version" + } for col_id, visible in COLUMNS_PREFERENCES_DEFAULTS.items(): - logger.debug(f" Column {col_id}: {'visible' if visible else 'hidden'}") + col_name = column_names.get(col_id, f"Column {col_id}") + logger.debug(f" {col_name} (#{col_id}): {'visible' if visible else 'hidden'}") # Prepare context data for templates data = { From 2f4b42b7e7f9ac493ed8f69a2692716eca49fb92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 09:38:47 +0200 Subject: [PATCH 07/15] Update README.md --- src/common/core/letsencrypt/README.md | 156 ++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 19 deletions(-) diff --git a/src/common/core/letsencrypt/README.md b/src/common/core/letsencrypt/README.md index 053a202cec..f81dab023e 100644 --- a/src/common/core/letsencrypt/README.md +++ b/src/common/core/letsencrypt/README.md @@ -1,13 +1,13 @@ -The Let's Encrypt plugin simplifies SSL/TLS certificate management by automating the creation, renewal, and configuration of free certificates from Let's Encrypt. This feature enables secure HTTPS connections for your websites without the complexity of manual certificate management, reducing both cost and administrative overhead. +The Let's Encrypt plugin simplifies SSL/TLS certificate management by automating the creation, renewal, and configuration of free certificates from multiple certificate authorities. This feature enables secure HTTPS connections for your websites without the complexity of manual certificate management, reducing both cost and administrative overhead. **How it works:** 1. When enabled, BunkerWeb automatically detects the domains configured for your website. -2. BunkerWeb requests free SSL/TLS certificates from Let's Encrypt's certificate authority. +2. BunkerWeb requests free SSL/TLS certificates from supported certificate authorities (Let's Encrypt, ZeroSSL). 3. Domain ownership is verified through either HTTP challenges (proving you control the website) or DNS challenges (proving you control your domain's DNS). 4. Certificates are automatically installed and configured for your domains. 5. BunkerWeb handles certificate renewals in the background before expiration, ensuring continuous HTTPS availability. -6. The entire process is fully automated, requiring minimal intervention after the initial setup. +6. The entire process is fully automated with intelligent retry mechanisms, requiring minimal intervention after the initial setup. !!! info "Prerequisites" To use this feature, ensure that proper DNS **A records** are configured for each domain, pointing to the public IP(s) where BunkerWeb is accessible. Without correct DNS configuration, the domain verification process will fail. @@ -17,15 +17,24 @@ The Let's Encrypt plugin simplifies SSL/TLS certificate management by automating Follow these steps to configure and use the Let's Encrypt feature: 1. **Enable the feature:** Set the `AUTO_LETS_ENCRYPT` setting to `yes` to enable automatic certificate issuance and renewal. -2. **Provide contact email:** Enter your email address using the `EMAIL_LETS_ENCRYPT` setting to receive important notifications about your certificates. -3. **Choose challenge type:** Select either `http` or `dns` verification with the `LETS_ENCRYPT_CHALLENGE` setting. -4. **Configure DNS provider:** If using DNS challenges, specify your DNS provider and credentials. -5. **Select certificate profile:** Choose your preferred certificate profile using the `LETS_ENCRYPT_PROFILE` setting (classic, tlsserver, or shortlived). -6. **Let BunkerWeb handle the rest:** Once configured, certificates are automatically issued, installed, and renewed as needed. +2. **Choose certificate authority:** Select your preferred CA using `ACME_SSL_CA_PROVIDER` (letsencrypt or zerossl). +3. **Provide contact email:** Enter your email address using the `EMAIL_LETS_ENCRYPT` setting to receive important notifications about your certificates. +4. **Choose challenge type:** Select either `http` or `dns` verification with the `LETS_ENCRYPT_CHALLENGE` setting. +5. **Configure DNS provider:** If using DNS challenges, specify your DNS provider and credentials. +6. **Select certificate profile:** Choose your preferred certificate profile using the `LETS_ENCRYPT_PROFILE` setting (classic, tlsserver, or shortlived). +7. **Configure API keys:** For ZeroSSL, provide your API key using `ACME_ZEROSSL_API_KEY`. +8. **Let BunkerWeb handle the rest:** Once configured, certificates are automatically issued, installed, and renewed as needed with intelligent retry mechanisms. + +!!! tip "Certificate Authorities" + The plugin supports multiple certificate authorities: + - **Let's Encrypt**: Free, widely trusted, 90-day certificates + - **ZeroSSL**: Free alternative with competitive rate limits, supports EAB (External Account Binding) + + ZeroSSL requires an API key for automated EAB credential generation. Without an API key, you can manually provide EAB credentials using `ACME_ZEROSSL_EAB_KID` and `ACME_ZEROSSL_EAB_HMAC_KEY`. !!! tip "Certificate Profiles" - Let's Encrypt provides different certificate profiles for different use cases: - - **classic**: General-purpose certificates with 90-day validity (default) + Let's Encrypt and ZeroSSL provide different certificate profiles for different use cases: + - **classic**: General-purpose certificates with 90-day validity (default, widest compatibility) - **tlsserver**: Optimized for TLS server authentication with 90-day validity and smaller payload - **shortlived**: Enhanced security with 7-day validity for automated environments - **custom**: If your ACME server supports a different profile, set it using `LETS_ENCRYPT_CUSTOM_PROFILE`. @@ -33,29 +42,49 @@ Follow these steps to configure and use the Let's Encrypt feature: !!! info "Profile Availability" Note that the `tlsserver` and `shortlived` profiles may not be available in all environments or with all ACME clients at this time. The `classic` profile has the widest compatibility and is recommended for most users. If a selected profile is not available, the system will automatically fall back to the `classic` profile. +### Advanced Security Features + +The plugin includes several advanced security and validation features: + +- **Public Suffix List (PSL) Validation**: Automatically prevents certificate requests for domains on the PSL (controlled by `LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES`) +- **CAA Record Validation**: Checks DNS CAA records to ensure the selected certificate authority is authorized for your domains +- **IP Address Validation**: For HTTP challenges, validates that domain DNS records point to your server's public IP addresses +- **Retry Mechanisms**: Intelligent retry with exponential backoff for failed certificate generation attempts +- **Certificate Key Types**: Supports both RSA and ECDSA keys with provider-specific optimizations + ### Configuration Settings | Setting | Default | Context | Multiple | Description | | ---------------------------------- | ------------------------ | --------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `AUTO_LETS_ENCRYPT` | `no` | multisite | no | **Enable Let's Encrypt:** Set to `yes` to enable automatic certificate issuance and renewal. | +| `ACME_SSL_CA_PROVIDER` | `letsencrypt` | multisite | no | **Certificate Authority:** Select certificate authority (`letsencrypt` or `zerossl`). | +| `ACME_ZEROSSL_API_KEY` | | multisite | no | **ZeroSSL API Key:** Required for automated ZeroSSL certificate generation and EAB credential setup. | +| `ACME_ZEROSSL_EAB_KID` | | multisite | no | **ZeroSSL EAB Key ID:** Manual EAB key identifier (alternative to API key). Used with `ACME_ZEROSSL_EAB_HMAC_KEY`. | +| `ACME_ZEROSSL_EAB_HMAC_KEY` | | multisite | no | **ZeroSSL EAB HMAC Key:** Manual EAB HMAC key (alternative to API key). Used with `ACME_ZEROSSL_EAB_KID`. | | `LETS_ENCRYPT_PASSTHROUGH` | `no` | multisite | no | **Pass Through Let's Encrypt:** Set to `yes` to pass through Let's Encrypt requests to the web server. This is useful when BunkerWeb is behind another reverse proxy handling SSL. | -| `EMAIL_LETS_ENCRYPT` | `contact@{FIRST_SERVER}` | multisite | no | **Contact Email:** Email address that is used for Let's Encrypt notifications and is included in certificates. | +| `EMAIL_LETS_ENCRYPT` | `contact@{FIRST_SERVER}` | multisite | no | **Contact Email:** Email address that is used for certificate notifications and is included in certificates. | | `LETS_ENCRYPT_CHALLENGE` | `http` | multisite | no | **Challenge Type:** Method used to verify domain ownership. Options: `http` or `dns`. | | `LETS_ENCRYPT_DNS_PROVIDER` | | multisite | no | **DNS Provider:** When using DNS challenges, the DNS provider to use (e.g., cloudflare, route53, digitalocean). | | `LETS_ENCRYPT_DNS_PROPAGATION` | `default` | multisite | no | **DNS Propagation:** The time to wait for DNS propagation in seconds. If no value is provided, the provider's default propagation time is used. | | `LETS_ENCRYPT_DNS_CREDENTIAL_ITEM` | | multisite | yes | **Credential Item:** Configuration items for DNS provider authentication (e.g., `cloudflare_api_token 123456`). Values can be raw text, base64 encoded, or a JSON object. | | `USE_LETS_ENCRYPT_WILDCARD` | `no` | multisite | no | **Wildcard Certificates:** When set to `yes`, creates wildcard certificates for all domains. Only available with DNS challenges. | -| `USE_LETS_ENCRYPT_STAGING` | `no` | multisite | no | **Use Staging:** When set to `yes`, uses Let's Encrypt's staging environment for testing. Staging has higher rate limits but produces certificates that are not trusted by browsers. | +| `USE_LETS_ENCRYPT_STAGING` | `no` | multisite | no | **Use Staging:** When set to `yes`, uses staging environment for testing. Staging has higher rate limits but produces certificates that are not trusted by browsers. | | `LETS_ENCRYPT_CLEAR_OLD_CERTS` | `no` | global | no | **Clear Old Certificates:** When set to `yes`, removes old certificates that are no longer needed during renewal. | | `LETS_ENCRYPT_PROFILE` | `classic` | multisite | no | **Certificate Profile:** Select the certificate profile to use. Options: `classic` (general-purpose), `tlsserver` (optimized for TLS servers), or `shortlived` (7-day certificates). | | `LETS_ENCRYPT_CUSTOM_PROFILE` | | multisite | no | **Custom Certificate Profile:** Enter a custom certificate profile if your ACME server supports non-standard profiles. This overrides `LETS_ENCRYPT_PROFILE` if set. | -| `LETS_ENCRYPT_MAX_RETRIES` | `3` | multisite | no | **Maximum Retries:** Number of times to retry certificate generation on failure. Set to `0` to disable retries. Useful for handling temporary network issues or API rate limits. | +| `LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES` | `yes` | multisite | no | **Disable Public Suffixes:** When set to `yes`, prevents certificate requests for domains matching the Public Suffix List (recommended for security). | +| `LETS_ENCRYPT_MAX_RETRIES` | `0` | multisite | no | **Maximum Retries:** Number of times to retry certificate generation on failure. Set to `0` to disable retries. Uses exponential backoff (30s, 60s, 120s, etc. up to 300s max). | +| `ACME_SKIP_CAA_CHECK` | `no` | global | no | **Skip CAA Validation:** Set to `yes` to skip DNS CAA record validation. Use with caution as CAA records provide important security controls. | +| `ACME_HTTP_STRICT_IP_CHECK` | `no` | global | no | **Strict IP Check:** When set to `yes`, rejects HTTP challenge certificates if domain IP doesn't match server IP. Useful for preventing misconfigurations. | !!! info "Information and behavior" - The `LETS_ENCRYPT_DNS_CREDENTIAL_ITEM` setting is a multiple setting and can be used to set multiple items for the DNS provider. The items will be saved as a cache file, and Certbot will read the credentials from it. - If no `LETS_ENCRYPT_DNS_PROPAGATION` setting is provided, the provider's default propagation time is used. - Full Let's Encrypt automation using the `http` challenge works in stream mode as long as you open the `80/tcp` port from the outside. Use the `LISTEN_STREAM_PORT_SSL` setting to choose your listening SSL/TLS port. - - If `LETS_ENCRYPT_PASSTHROUGH` is set to `yes`, BunkerWeb will not handle the ACME challenge requests itself but will pass them to the backend web server. This is useful in scenarios where BunkerWeb is acting as a reverse proxy in front of another server that is configured to handle Let's Encrypt challenges + - If `LETS_ENCRYPT_PASSTHROUGH` is set to `yes`, BunkerWeb will not handle the ACME challenge requests itself but will pass them to the backend web server. This is useful in scenarios where BunkerWeb is acting as a reverse proxy in front of another server that is configured to handle Let's Encrypt challenges. + - The plugin automatically validates external IP addresses for HTTP challenges and can optionally enforce strict IP matching. + - CAA record validation ensures only authorized certificate authorities can issue certificates for your domains. + - Public Suffix List validation prevents certificate requests for domains like `.com` or `.co.uk` that could never be validated. !!! tip "HTTP vs. DNS Challenges" **HTTP Challenges** are easier to set up and work well for most websites: @@ -63,6 +92,7 @@ Follow these steps to configure and use the Let's Encrypt feature: - Requires your website to be publicly accessible on port 80 - Automatically configured by BunkerWeb - Cannot be used for wildcard certificates + - Includes IP address validation for additional security **DNS Challenges** offer more flexibility and are required for wildcard certificates: @@ -70,13 +100,13 @@ Follow these steps to configure and use the Let's Encrypt feature: - Requires DNS provider API credentials - Required for wildcard certificates (e.g., *.example.com) - Useful when port 80 is blocked or unavailable + - Supports advanced wildcard domain grouping !!! warning "Wildcard certificates" Wildcard certificates are only available with DNS challenges. If you want to use them, you must set the `USE_LETS_ENCRYPT_WILDCARD` setting to `yes` and properly configure your DNS provider credentials. !!! warning "Rate Limits" - Let's Encrypt imposes rate limits on certificate issuance. When testing configurations, use the staging environment by setting `USE_LETS_ENCRYPT_STAGING` to `yes` to avoid hitting production rate limits. Staging certificates are not trusted by browsers but are useful for validating your setup. - + Certificate authorities impose rate limits on certificate issuance. When testing configurations, use the staging environment by setting `USE_LETS_ENCRYPT_STAGING` to `yes` to avoid hitting production rate limits. Staging certificates are not trusted by browsers but are useful for validating your setup. ### Supported DNS Providers @@ -103,9 +133,19 @@ The Let's Encrypt plugin supports a wide range of DNS providers for DNS challeng | `sakuracloud` | Sakura Cloud | `api_token`
    `api_secret` | | [Documentation](https://certbot-dns-sakuracloud.readthedocs.io/en/stable/) | | `scaleway` | Scaleway | `application_token` | | [Documentation](https://github.com/vanonox/certbot-dns-scaleway/blob/main/README.rst) | +### Certificate Key Types and Optimization + +The plugin automatically selects optimal certificate key types based on the certificate authority and DNS provider: + +- **ECDSA Keys**: Used by default for most providers + - Let's Encrypt: P-256 curve (secp256r1) for optimal performance + - ZeroSSL: P-384 curve (secp384r1) for enhanced security +- **RSA Keys**: Used for specific providers that require them + - Infomaniak and IONOS: RSA-4096 for compatibility + ### Example Configurations -=== "Basic HTTP Challenge" +=== "Basic HTTP Challenge with Let's Encrypt" Simple configuration using HTTP challenges for a single domain: @@ -113,6 +153,32 @@ The Let's Encrypt plugin supports a wide range of DNS providers for DNS challeng AUTO_LETS_ENCRYPT: "yes" EMAIL_LETS_ENCRYPT: "admin@example.com" LETS_ENCRYPT_CHALLENGE: "http" + ACME_SSL_CA_PROVIDER: "letsencrypt" + ``` + +=== "ZeroSSL with API Key" + + Configuration using ZeroSSL with automated EAB setup: + + ```yaml + AUTO_LETS_ENCRYPT: "yes" + EMAIL_LETS_ENCRYPT: "admin@example.com" + LETS_ENCRYPT_CHALLENGE: "http" + ACME_SSL_CA_PROVIDER: "zerossl" + ACME_ZEROSSL_API_KEY: "your-zerossl-api-key" + ``` + +=== "ZeroSSL with Manual EAB Credentials" + + Configuration using ZeroSSL with manually provided EAB credentials: + + ```yaml + AUTO_LETS_ENCRYPT: "yes" + EMAIL_LETS_ENCRYPT: "admin@example.com" + LETS_ENCRYPT_CHALLENGE: "http" + ACME_SSL_CA_PROVIDER: "zerossl" + ACME_ZEROSSL_EAB_KID: "your-eab-kid" + ACME_ZEROSSL_EAB_HMAC_KEY: "your-eab-hmac-key" ``` === "Cloudflare DNS with Wildcard" @@ -141,9 +207,24 @@ The Let's Encrypt plugin supports a wide range of DNS providers for DNS challeng LETS_ENCRYPT_DNS_CREDENTIAL_ITEM_2: "aws_secret_access_key YOUR_SECRET_KEY" ``` -=== "Testing with Staging Environment and Retries" +=== "Production with Enhanced Security" + + Configuration with all security features enabled and retry mechanisms: + + ```yaml + AUTO_LETS_ENCRYPT: "yes" + EMAIL_LETS_ENCRYPT: "admin@example.com" + LETS_ENCRYPT_CHALLENGE: "http" + ACME_SSL_CA_PROVIDER: "letsencrypt" + LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES: "yes" + ACME_HTTP_STRICT_IP_CHECK: "yes" + LETS_ENCRYPT_MAX_RETRIES: "3" + LETS_ENCRYPT_PROFILE: "tlsserver" + ``` + +=== "Testing with Staging Environment" - Configuration for testing setup with the staging environment and enhanced retry settings: + Configuration for testing setup with the staging environment: ```yaml AUTO_LETS_ENCRYPT: "yes" @@ -182,3 +263,40 @@ The Let's Encrypt plugin supports a wide range of DNS providers for DNS challeng LETS_ENCRYPT_DNS_CREDENTIAL_ITEM_5: "client_id your-client-id" LETS_ENCRYPT_DNS_CREDENTIAL_ITEM_6: "client_x509_cert_url your-cert-url" ``` + +### Troubleshooting + +**Common Issues and Solutions:** + +1. **Certificate generation fails with rate limits** + - Use staging environment: `USE_LETS_ENCRYPT_STAGING: "yes"` + - Enable retries: `LETS_ENCRYPT_MAX_RETRIES: "3"` + +2. **HTTP challenge fails** + - Verify domain DNS points to your server IP + - Enable strict IP checking: `ACME_HTTP_STRICT_IP_CHECK: "yes"` + - Check firewall allows port 80 access + +3. **DNS challenge fails** + - Verify DNS provider credentials are correct + - Increase propagation time: `LETS_ENCRYPT_DNS_PROPAGATION: "300"` + - Check DNS provider API rate limits + +4. **CAA validation errors** + - Update CAA records to authorize your chosen certificate authority + - Temporarily skip CAA checking: `ACME_SKIP_CAA_CHECK: "yes"` + +5. **ZeroSSL EAB issues** + - Ensure API key is valid and has correct permissions + - Try manual EAB credentials if API setup fails + - Check ZeroSSL account has ACME access enabled + +**Debug Information:** + +Enable debug logging by setting `LOG_LEVEL: "DEBUG"` to get detailed information about: +- Certificate generation process +- DNS validation steps +- HTTP challenge deployment +- CAA record checking +- IP address validation +- Retry attempts and backoff timing \ No newline at end of file From cfdadb2247064c3c1bd72b401093a0162f926190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 11:28:04 +0200 Subject: [PATCH 08/15] fix Pylance error messages --- .../core/letsencrypt/jobs/certbot-deploy.py | 1 + .../core/letsencrypt/jobs/certbot-new.py | 1247 ++++++++++------- .../core/letsencrypt/jobs/certbot-renew.py | 7 +- src/common/core/letsencrypt/ui/actions.py | 157 ++- .../letsencrypt/ui/blueprints/letsencrypt.py | 131 +- 5 files changed, 938 insertions(+), 605 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index b0ff874c0e..c5eaf7c9a2 100644 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -3,6 +3,7 @@ from io import BytesIO from os import getenv, sep from os.path import join +from pathlib import Path from sys import exit as sys_exit, path as sys_path from tarfile import open as tar_open from traceback import format_exc diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index 75133d1f16..c89737447c 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -17,7 +17,7 @@ from sys import exit as sys_exit, path as sys_path from time import sleep from traceback import format_exc -from typing import Dict, Literal, Type, Union +from typing import Dict, Literal, Optional, Type, Union for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: @@ -69,7 +69,7 @@ LOGS_DIR = join(sep, "var", "log", "bunkerweb", "letsencrypt") PSL_URL = "https://publicsuffix.org/list/public_suffix_list.dat" -PSL_STATIC_FILE = "public_suffix_list.dat" +PSL_STATIC_FILE = Path("public_suffix_list.dat") # ZeroSSL Configuration ZEROSSL_ACME_SERVER = "https://acme.zerossl.com/v2/DV90" @@ -86,15 +86,16 @@ def load_public_suffix_list(job): # Returns cached version if available and fresh (less than 1 day old). # # Args: - # job - Job instance for caching operations + # job: Job instance for caching operations # # Returns: - # list - Lines from the public suffix list file + # list: Lines from the public suffix list file if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Loading public suffix list from cache or {PSL_URL}") LOGGER.debug("Checking if cached PSL is available and fresh") - job_cache = job.get_cache(PSL_STATIC_FILE, with_info=True, with_data=True) + job_cache = job.get_cache(PSL_STATIC_FILE.name, with_info=True, + with_data=True) if ( isinstance(job_cache, dict) and job_cache.get("last_update") @@ -104,7 +105,9 @@ def load_public_suffix_list(job): ): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Using cached public suffix list") - LOGGER.debug(f"Cache age: {(datetime.now().astimezone().timestamp() - job_cache['last_update']) / 3600:.1f} hours") + cache_age_hours = ((datetime.now().astimezone().timestamp() - + job_cache['last_update']) / 3600) + LOGGER.debug(f"Cache age: {cache_age_hours:.1f} hours") return job_cache["data"].decode("utf-8").splitlines() try: @@ -120,9 +123,11 @@ def load_public_suffix_list(job): LOGGER.debug(f"Downloaded PSL successfully, {len(content)} bytes") LOGGER.debug(f"PSL contains {len(content.splitlines())} lines") - cached, err = JOB.cache_file(PSL_STATIC_FILE, content.encode("utf-8")) + cached, err = JOB.cache_file(PSL_STATIC_FILE.name, + content.encode("utf-8")) if not cached: - LOGGER.error(f"Error while saving public suffix list to cache: {err}") + LOGGER.error(f"Error while saving public suffix list to cache: " + f"{err}") else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("PSL successfully cached for future use") @@ -137,7 +142,8 @@ def load_public_suffix_list(job): if PSL_STATIC_FILE.exists(): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using existing static PSL file: {PSL_STATIC_FILE}") + LOGGER.debug(f"Using existing static PSL file: " + f"{PSL_STATIC_FILE}") with PSL_STATIC_FILE.open("r", encoding="utf-8") as f: return f.read().splitlines() @@ -152,10 +158,10 @@ def parse_psl(psl_lines): # exceptions (lines starting with !), and regular rules. # # Args: - # psl_lines - List of lines from the PSL file + # psl_lines: List of lines from the PSL file # # Returns: - # dict - Contains 'rules' and 'exceptions' sets + # dict: Contains 'rules' and 'exceptions' sets if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Parsing {len(psl_lines)} PSL lines") LOGGER.debug("Processing rules, exceptions, and filtering comments") @@ -179,8 +185,10 @@ def parse_psl(psl_lines): rules.add(line) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsed {len(rules)} rules and {len(exceptions)} exceptions") - LOGGER.debug(f"Skipped {comments_skipped} comments and {empty_lines_skipped} empty lines") + LOGGER.debug(f"Parsed {len(rules)} rules and {len(exceptions)} " + f"exceptions") + LOGGER.debug(f"Skipped {comments_skipped} comments and " + f"{empty_lines_skipped} empty lines") LOGGER.debug("PSL parsing completed successfully") return {"rules": rules, "exceptions": exceptions} @@ -192,28 +200,30 @@ def is_domain_blacklisted(domain, psl): # Public Suffix List rules and exceptions. # # Args: - # domain - Domain name to check - # psl - Parsed PSL data (dict with 'rules' and 'exceptions') + # domain: Domain name to check + # psl: Parsed PSL data (dict with 'rules' and 'exceptions') # # Returns: - # bool - True if domain is blacklisted + # bool: True if domain is blacklisted domain = domain.lower().strip(".") labels = domain.split(".") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking domain {domain} against PSL rules") LOGGER.debug(f"Domain has {len(labels)} labels: {labels}") - LOGGER.debug(f"PSL contains {len(psl['rules'])} rules and {len(psl['exceptions'])} exceptions") + LOGGER.debug(f"PSL contains {len(psl['rules'])} rules and " + f"{len(psl['exceptions'])} exceptions") for i in range(len(labels)): - candidate = ".".join(labels[i:]) + candidate = ".".join(str(label) for label in labels[i:]) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking candidate: {candidate}") if candidate in psl["exceptions"]: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} allowed by PSL exception {candidate}") + LOGGER.debug(f"Domain {domain} allowed by PSL exception " + f"{candidate}") return False if candidate in psl["rules"]: @@ -223,29 +233,35 @@ def is_domain_blacklisted(domain, psl): if i == 0: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - exact PSL rule match") + LOGGER.debug(f"Domain {domain} blacklisted - exact PSL " + f"rule match") return True if i == 0 and domain.startswith("*."): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Wildcard domain {domain} blacklisted - exact PSL rule match") + LOGGER.debug(f"Wildcard domain {domain} blacklisted - " + f"exact PSL rule match") return True if i == 0 or (i == 1 and labels[0] == "*"): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - PSL rule violation") + LOGGER.debug(f"Domain {domain} blacklisted - PSL rule " + f"violation") return True if len(labels[i:]) == len(labels): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - full label match") + LOGGER.debug(f"Domain {domain} blacklisted - full label " + f"match") return True wildcard_candidate = f"*.{candidate}" if wildcard_candidate in psl["rules"]: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found PSL wildcard rule match: {wildcard_candidate}") + LOGGER.debug(f"Found PSL wildcard rule match: " + f"{wildcard_candidate}") if len(labels[i:]) == 2: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - wildcard PSL rule match") + LOGGER.debug(f"Domain {domain} blacklisted - wildcard " + f"PSL rule match") return True if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -259,17 +275,19 @@ def get_certificate_authority_config(ca_provider, staging=False): # certificate authority and environment (staging/production). # # Args: - # ca_provider - Certificate authority name ('zerossl' or 'letsencrypt') - # staging - Whether to use staging environment + # ca_provider: Certificate authority name ('zerossl' or 'letsencrypt') + # staging: Whether to use staging environment # # Returns: - # dict - Server URL and CA name + # dict: Server URL and CA name if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Getting CA config for {ca_provider}, staging={staging}") + LOGGER.debug(f"Getting CA config for {ca_provider}, " + f"staging={staging}") if ca_provider.lower() == "zerossl": config = { - "server": ZEROSSL_STAGING_SERVER if staging else ZEROSSL_ACME_SERVER, + "server": (ZEROSSL_STAGING_SERVER if staging + else ZEROSSL_ACME_SERVER), "name": "ZeroSSL" } else: # Default to Let's Encrypt @@ -291,11 +309,11 @@ def setup_zerossl_eab_credentials(email, api_key=None): # ACME certificate issuance with ZeroSSL. # # Args: - # email - Email address for the account - # api_key - ZeroSSL API key + # email: Email address for the account + # api_key: ZeroSSL API key # # Returns: - # tuple - (eab_kid, eab_hmac_key) or (None, None) on failure + # tuple: (eab_kid, eab_hmac_key) or (None, None) on failure LOGGER.info(f"Setting up ZeroSSL EAB credentials for email: {email}") if not api_key: @@ -325,7 +343,8 @@ def setup_zerossl_eab_credentials(email, api_key=None): ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL API response status: {response.status_code}") + LOGGER.debug(f"ZeroSSL API response status: " + f"{response.status_code}") LOGGER.debug(f"Response headers: {dict(response.headers)}") LOGGER.info(f"ZeroSSL API response status: {response.status_code}") @@ -341,14 +360,17 @@ def setup_zerossl_eab_credentials(email, api_key=None): if "eab_kid" in eab_data and "eab_hmac_key" in eab_data: eab_kid = eab_data.get("eab_kid") eab_hmac_key = eab_data.get("eab_hmac_key") - LOGGER.info(f"✓ Successfully obtained EAB credentials from ZeroSSL") - LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") - LOGGER.info( - f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}..." - ) + LOGGER.info(f"✓ Successfully obtained EAB credentials from " + f"ZeroSSL") + kid_display = f"{eab_kid[:10]}..." if eab_kid else "None" + hmac_display = (f"{eab_hmac_key[:10]}..." if eab_hmac_key + else "None") + LOGGER.info(f"EAB Kid: {kid_display}") + LOGGER.info(f"EAB HMAC Key: {hmac_display}") return eab_kid, eab_hmac_key else: - LOGGER.error(f"❌ Invalid ZeroSSL API response format: {eab_data}") + LOGGER.error(f"❌ Invalid ZeroSSL API response format: " + f"{eab_data}") return None, None else: # Try alternative endpoint if first one fails @@ -374,17 +396,17 @@ def setup_zerossl_eab_credentials(email, api_key=None): ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Alternative ZeroSSL API response status: {response.status_code}" - ) - LOGGER.info( - f"Alternative ZeroSSL API response status: {response.status_code}" - ) + alt_status = response.status_code + LOGGER.debug(f"Alternative ZeroSSL API response status: " + f"{alt_status}") + LOGGER.info(f"Alternative ZeroSSL API response status: " + f"{response.status_code}") response.raise_for_status() eab_data = response.json() if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Alternative ZeroSSL API response data: {eab_data}") + LOGGER.debug(f"Alternative ZeroSSL API response data: " + f"{eab_data}") LOGGER.info(f"Alternative ZeroSSL API response data: {eab_data}") if eab_data.get("success"): @@ -394,10 +416,11 @@ def setup_zerossl_eab_credentials(email, api_key=None): "✓ Successfully obtained EAB credentials from ZeroSSL " "(alternative endpoint)" ) - LOGGER.info(f"EAB Kid: {eab_kid[:10] if eab_kid else 'None'}...") - LOGGER.info( - f"EAB HMAC Key: {eab_hmac_key[:10] if eab_hmac_key else 'None'}..." - ) + kid_display = f"{eab_kid[:10]}..." if eab_kid else "None" + hmac_display = (f"{eab_hmac_key[:10]}..." if eab_hmac_key + else "None") + LOGGER.info(f"EAB Kid: {kid_display}") + LOGGER.info(f"EAB HMAC Key: {hmac_display}") return eab_kid, eab_hmac_key else: LOGGER.error(f"❌ ZeroSSL EAB registration failed: {eab_data}") @@ -426,10 +449,10 @@ def get_caa_records(domain): # Returns None if dig command is not available. # # Args: - # domain - Domain name to query + # domain: Domain name to query # # Returns: - # list or None - List of CAA record dicts or None if unavailable + # list or None: List of CAA record dicts or None if unavailable # Check if dig command is available if not which("dig"): @@ -485,11 +508,15 @@ def get_caa_records(domain): }) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsed CAA record: flags={flags}, tag={tag}, value={value}") + LOGGER.debug(f"Parsed CAA record: flags={flags}, " + f"tag={tag}, value={value}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Successfully parsed {len(caa_records)} CAA records for domain {domain}") - LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain {domain}") + record_count = len(caa_records) + LOGGER.debug(f"Successfully parsed {record_count} CAA records " + f"for domain {domain}") + LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain " + f"{domain}") return caa_records else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -516,12 +543,12 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # certificates for the domain according to CAA DNS records. # # Args: - # domain - Domain name to check - # ca_provider - Certificate authority provider name - # is_wildcard - Whether this is for a wildcard certificate + # domain: Domain name to check + # ca_provider: Certificate authority provider name + # is_wildcard: Whether this is for a wildcard certificate # # Returns: - # bool - True if CA is authorized or no CAA restrictions exist + # bool: True if CA is authorized or no CAA restrictions exist if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( f"Checking CAA authorization for domain: {domain}, " @@ -543,23 +570,26 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): if not allowed_identifiers: LOGGER.warning(f"Unknown CA provider for CAA check: {ca_provider}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Returning True for unknown CA provider (conservative approach)") + LOGGER.debug("Returning True for unknown CA provider " + "(conservative approach)") return True # Allow unknown providers (conservative approach) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CA identifiers for {ca_provider}: {allowed_identifiers}") + LOGGER.debug(f"CA identifiers for {ca_provider}: " + f"{allowed_identifiers}") # Check CAA records for the domain and parent domains check_domain = domain.lstrip("*.") domain_parts = check_domain.split(".") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Will check CAA records for domain chain: {check_domain}") + LOGGER.debug(f"Will check CAA records for domain chain: " + f"{check_domain}") LOGGER.debug(f"Domain parts: {domain_parts}") LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") for i in range(len(domain_parts)): - current_domain = ".".join(domain_parts[i:]) + current_domain = ".".join(str(part) for part in domain_parts[i:]) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking CAA records for: {current_domain}") @@ -568,7 +598,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): if caa_records is None: # dig not available, skip CAA check - LOGGER.info("CAA record checking skipped (dig command not available)") + LOGGER.info("CAA record checking skipped (dig command not " + "available)") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Returning True due to unavailable dig command") return True @@ -589,35 +620,46 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Log found records if issue_records: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA issue records: {', '.join(issue_records)}") - LOGGER.info(f"CAA issue records: {', '.join(issue_records)}") + LOGGER.debug(f"CAA issue records: " + f"{', '.join(str(record) for record in issue_records)}") + LOGGER.info(f"CAA issue records: " + f"{', '.join(str(record) for record in issue_records)}") if issuewild_records: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA issuewild records: {', '.join(issuewild_records)}") - LOGGER.info(f"CAA issuewild records: {', '.join(issuewild_records)}") + LOGGER.debug(f"CAA issuewild records: " + f"{', '.join(str(record) for record in issuewild_records)}") + LOGGER.info(f"CAA issuewild records: " + f"{', '.join(str(record) for record in issuewild_records)}") # Check authorization based on certificate type if is_wildcard: - # For wildcard certificates, check issuewild first, then fall back to issue - check_records = issuewild_records if issuewild_records else issue_records - record_type = "issuewild" if issuewild_records else "issue" + # For wildcard certificates, check issuewild first, + # then fall back to issue + check_records = (issuewild_records if issuewild_records + else issue_records) + record_type = ("issuewild" if issuewild_records + else "issue") else: # For regular certificates, check issue records check_records = issue_records record_type = "issue" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using CAA {record_type} records for authorization check") + LOGGER.debug(f"Using CAA {record_type} records for " + f"authorization check") LOGGER.debug(f"Records to check: {check_records}") - LOGGER.info(f"Using CAA {record_type} records for authorization check") + LOGGER.info(f"Using CAA {record_type} records for authorization " + f"check") if not check_records: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( - f"No relevant CAA {record_type} records found for {current_domain}" + f"No relevant CAA {record_type} records found for " + f"{current_domain}" ) LOGGER.info( - f"No relevant CAA {record_type} records found for {current_domain}" + f"No relevant CAA {record_type} records found for " + f"{current_domain}" ) continue @@ -625,37 +667,43 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): authorized = False if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + identifier_list = ', '.join(str(id) for id in allowed_identifiers) LOGGER.debug( f"Checking authorization for CA identifiers: " - f"{', '.join(allowed_identifiers)}" + f"{identifier_list}" ) + identifier_list = ', '.join(str(id) for id in allowed_identifiers) LOGGER.info( f"Checking authorization for CA identifiers: " - f"{', '.join(allowed_identifiers)}" + f"{identifier_list}" ) for identifier in allowed_identifiers: for record in check_records: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Comparing identifier '{identifier}' with record '{record}'") + LOGGER.debug(f"Comparing identifier '{identifier}' " + f"with record '{record}'") # Handle explicit deny (empty value or semicolon) if record == ";" or record.strip() == "": LOGGER.warning( - f"CAA {record_type} record explicitly denies all CAs" + f"CAA {record_type} record explicitly denies " + f"all CAs" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Found explicit deny record - authorization failed") + LOGGER.debug("Found explicit deny record - " + "authorization failed") return False # Check for CA authorization if identifier in record: authorized = True LOGGER.info( - f"✓ CA {ca_provider} ({identifier}) authorized by " - f"CAA {record_type} record" + f"✓ CA {ca_provider} ({identifier}) authorized " + f"by CAA {record_type} record" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Authorization found: {identifier} in {record}") + LOGGER.debug(f"Authorization found: {identifier} " + f"in {record}") break if authorized: break @@ -665,33 +713,41 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"❌ CA {ca_provider} is NOT authorized by " f"CAA {record_type} records" ) + allowed_list = ', '.join(str(record) for record in check_records) + identifier_list = ', '.join(str(id) for id in allowed_identifiers) LOGGER.error( f"Domain {current_domain} CAA {record_type} allows: " - f"{', '.join(check_records)}" + f"{allowed_list}" ) LOGGER.error( - f"But {ca_provider} uses: {', '.join(allowed_identifiers)}" + f"But {ca_provider} uses: {identifier_list}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("CAA authorization failed - no matching identifiers") + LOGGER.debug("CAA authorization failed - no matching " + "identifiers") return False - # If we found CAA records and we're authorized, we can stop checking parent domains + # If we found CAA records and we're authorized, we can stop + # checking parent domains LOGGER.info(f"✓ CAA authorization successful for {domain}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("CAA authorization successful - stopping parent domain checks") + LOGGER.debug("CAA authorization successful - stopping parent " + "domain checks") return True # No CAA records found in the entire chain LOGGER.info( - f"No CAA records found for {check_domain} or parent domains - any CA allowed" + f"No CAA records found for {check_domain} or parent domains - " + f"any CA allowed" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No CAA records found in entire domain chain - allowing any CA") + LOGGER.debug("No CAA records found in entire domain chain - " + "allowing any CA") return True -def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", +def validate_domains_for_http_challenge(domains_list, + ca_provider="letsencrypt", is_wildcard=False): # Validate that all domains have valid A/AAAA records and CAA authorization # for HTTP challenge. @@ -699,21 +755,24 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", # domain in the list to ensure HTTP challenge will succeed. # # Args: - # domains_list - List of domain names to validate - # ca_provider - Certificate authority provider name - # is_wildcard - Whether this is for wildcard certificates + # domains_list: List of domain names to validate + # ca_provider: Certificate authority provider name + # is_wildcard: Whether this is for wildcard certificates # # Returns: - # bool - True if all domains are valid for HTTP challenge + # bool: True if all domains are valid for HTTP challenge if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + domain_count = len(domains_list) + domain_list = ', '.join(str(domain) for domain in domains_list) LOGGER.debug( - f"Validating {len(domains_list)} domains for HTTP challenge: " - f"{', '.join(domains_list)}" + f"Validating {domain_count} domains for HTTP challenge: " + f"{domain_list}" ) LOGGER.debug(f"CA provider: {ca_provider}, wildcard: {is_wildcard}") + domain_list = ', '.join(str(domain) for domain in domains_list) LOGGER.info( f"Validating {len(domains_list)} domains for HTTP challenge: " - f"{', '.join(domains_list)}" + f"{domain_list}" ) invalid_domains = [] caa_blocked_domains = [] @@ -722,18 +781,22 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", skip_caa_check = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA check will be {'skipped' if skip_caa_check else 'performed'}") + caa_status = 'skipped' if skip_caa_check else 'performed' + LOGGER.debug(f"CAA check will be {caa_status}") # Get external IPs once for all domain checks external_ips = get_external_ip() if external_ips: if external_ips.get("ipv4"): - LOGGER.info(f"Server external IPv4 address: {external_ips['ipv4']}") + LOGGER.info(f"Server external IPv4 address: " + f"{external_ips['ipv4']}") if external_ips.get("ipv6"): - LOGGER.info(f"Server external IPv6 address: {external_ips['ipv6']}") + LOGGER.info(f"Server external IPv6 address: " + f"{external_ips['ipv6']}") else: LOGGER.warning( - "Could not determine server external IP - skipping IP match validation" + "Could not determine server external IP - skipping IP match " + "validation" ) validation_passed = 0 @@ -760,42 +823,50 @@ def validate_domains_for_http_challenge(domains_list, ca_provider="letsencrypt", LOGGER.debug(f"CAA authorization failed for {domain}") else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") - LOGGER.info(f"CAA check skipped for {domain} (ACME_SKIP_CAA_CHECK=yes)") + LOGGER.debug(f"CAA check skipped for {domain} " + f"(ACME_SKIP_CAA_CHECK=yes)") + LOGGER.info(f"CAA check skipped for {domain} " + f"(ACME_SKIP_CAA_CHECK=yes)") validation_passed += 1 if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Validation passed for {domain}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Validation summary: {validation_passed} passed, {validation_failed} failed") + LOGGER.debug(f"Validation summary: {validation_passed} passed, " + f"{validation_failed} failed") # Report results if invalid_domains: + invalid_list = ', '.join(str(domain) for domain in invalid_domains) LOGGER.error( f"The following domains do not have valid A/AAAA records and " - f"cannot be used for HTTP challenge: {', '.join(invalid_domains)}" + f"cannot be used for HTTP challenge: {invalid_list}" ) LOGGER.error( - "Please ensure domains resolve to this server before requesting certificates" + "Please ensure domains resolve to this server before requesting " + "certificates" ) return False if caa_blocked_domains: + blocked_list = ', '.join(str(domain) for domain in caa_blocked_domains) LOGGER.error( - f"The following domains have CAA records that block {ca_provider}: " - f"{', '.join(caa_blocked_domains)}" + f"The following domains have CAA records that block " + f"{ca_provider}: {blocked_list}" ) LOGGER.error( - "Please update CAA records to authorize the certificate authority " - "or use a different CA" + "Please update CAA records to authorize the certificate " + "authority or use a different CA" ) - LOGGER.info("You can skip CAA checking by setting ACME_SKIP_CAA_CHECK=yes") + LOGGER.info("You can skip CAA checking by setting " + "ACME_SKIP_CAA_CHECK=yes") return False + valid_list = ', '.join(str(domain) for domain in domains_list) LOGGER.info( f"All domains have valid DNS records and CAA authorization for " - f"HTTP challenge: {', '.join(domains_list)}" + f"HTTP challenge: {valid_list}" ) return True @@ -806,7 +877,7 @@ def get_external_ip(): # IP addresses for DNS validation purposes. # # Returns: - # dict or None - Dict with 'ipv4' and 'ipv6' keys, or None if all fail + # dict or None: Dict with 'ipv4' and 'ipv6' keys, or None if all fail if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Getting external IP addresses for server") LOGGER.info("Getting external IP addresses for server") @@ -824,7 +895,7 @@ def get_external_ip(): "https://ipv6.jsonip.com" ] - external_ips = {"ipv4": None, "ipv6": None} + external_ips: Dict[str, Optional[str]] = {"ipv4": None, "ipv6": None} # Try to get IPv4 address if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -835,7 +906,8 @@ def get_external_ip(): for i, service in enumerate(ipv4_services): try: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Trying IPv4 service {i+1}/{len(ipv4_services)}: {service}") + service_num = f"{i+1}/{len(ipv4_services)}" + LOGGER.debug(f"Trying IPv4 service {service_num}: {service}") if "jsonip.com" in service: # This service returns JSON format @@ -857,11 +929,15 @@ def get_external_ip(): try: # Validate it's a proper IPv4 address getaddrinfo(ip, None, AF_INET) - external_ips["ipv4"] = ip + # Type-safe assignment + ipv4_addr: str = str(ip) + external_ips["ipv4"] = ipv4_addr if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Successfully obtained external IPv4 address: {ip}") - LOGGER.info(f"Successfully obtained external IPv4 address: {ip}") + LOGGER.debug(f"Successfully obtained external IPv4 " + f"address: {ipv4_addr}") + LOGGER.info(f"Successfully obtained external IPv4 " + f"address: {ipv4_addr}") break except gaierror: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -869,7 +945,8 @@ def get_external_ip(): continue except BaseException as e: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Failed to get IPv4 address from {service}: {e}") + LOGGER.debug(f"Failed to get IPv4 address from {service}: " + f"{e}") LOGGER.info(f"Failed to get IPv4 address from {service}: {e}") continue @@ -882,7 +959,8 @@ def get_external_ip(): for i, service in enumerate(ipv6_services): try: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Trying IPv6 service {i+1}/{len(ipv6_services)}: {service}") + service_num = f"{i+1}/{len(ipv6_services)}" + LOGGER.debug(f"Trying IPv6 service {service_num}: {service}") if "jsonip.com" in service: response = get(service, timeout=5) @@ -902,11 +980,15 @@ def get_external_ip(): try: # Validate it's a proper IPv6 address getaddrinfo(ip, None, AF_INET6) - external_ips["ipv6"] = ip + # Type-safe assignment + ipv6_addr: str = str(ip) + external_ips["ipv6"] = ipv6_addr if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Successfully obtained external IPv6 address: {ip}") - LOGGER.info(f"Successfully obtained external IPv6 address: {ip}") + LOGGER.debug(f"Successfully obtained external IPv6 " + f"address: {ipv6_addr}") + LOGGER.info(f"Successfully obtained external IPv6 " + f"address: {ipv6_addr}") break except gaierror: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -914,22 +996,25 @@ def get_external_ip(): continue except BaseException as e: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Failed to get IPv6 address from {service}: {e}") + LOGGER.debug(f"Failed to get IPv6 address from {service}: " + f"{e}") LOGGER.info(f"Failed to get IPv6 address from {service}: {e}") continue if not external_ips["ipv4"] and not external_ips["ipv6"]: LOGGER.warning( - "Could not determine external IP address (IPv4 or IPv6) from any service" + "Could not determine external IP address (IPv4 or IPv6) from " + "any service" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("All external IP services failed") return None + ipv4_status = external_ips['ipv4'] or 'not found' + ipv6_status = external_ips['ipv6'] or 'not found' LOGGER.info( f"External IP detection completed - " - f"IPv4: {external_ips['ipv4'] or 'not found'}, " - f"IPv6: {external_ips['ipv6'] or 'not found'}" + f"IPv4: {ipv4_status}, IPv6: {ipv6_status}" ) return external_ips @@ -940,11 +1025,11 @@ def check_domain_a_record(domain, external_ips=None): # IP addresses match the server's external IPs. # # Args: - # domain - Domain name to check - # external_ips - Dict with server's external IPv4/IPv6 addresses + # domain: Domain name to check + # external_ips: Dict with server's external IPv4/IPv6 addresses # # Returns: - # bool - True if domain has valid DNS records + # bool: True if domain has valid DNS records if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Checking DNS A/AAAA records for domain: {domain}") LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") @@ -953,13 +1038,16 @@ def check_domain_a_record(domain, external_ips=None): check_domain = domain.lstrip("*.") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking domain after wildcard removal: {check_domain}") + LOGGER.debug(f"Checking domain after wildcard removal: " + f"{check_domain}") # Attempt to resolve the domain to IP addresses result = getaddrinfo(check_domain, None) if result: - ipv4_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET] - ipv6_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET6] + ipv4_addresses = [addr[4][0] for addr in result + if addr[0] == AF_INET] + ipv6_addresses = [addr[4][0] for addr in result + if addr[0] == AF_INET6] if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"DNS resolution results:") @@ -967,31 +1055,36 @@ def check_domain_a_record(domain, external_ips=None): LOGGER.debug(f" IPv6 addresses: {ipv6_addresses}") if not ipv4_addresses and not ipv6_addresses: - LOGGER.warning(f"Domain {check_domain} has no A or AAAA records") + LOGGER.warning(f"Domain {check_domain} has no A or AAAA " + f"records") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No valid IP addresses found in DNS resolution") + LOGGER.debug("No valid IP addresses found in DNS " + "resolution") return False # Log found addresses if ipv4_addresses: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + ipv4_display = ', '.join(str(addr) for addr in ipv4_addresses[:3]) LOGGER.debug( f"Domain {check_domain} IPv4 A records: " - f"{', '.join(ipv4_addresses[:3])}" + f"{ipv4_display}" ) + ipv4_display = ', '.join(str(addr) for addr in ipv4_addresses[:3]) LOGGER.info( - f"Domain {check_domain} IPv4 A records: " - f"{', '.join(ipv4_addresses[:3])}" + f"Domain {check_domain} IPv4 A records: {ipv4_display}" ) if ipv6_addresses: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + ipv6_display = ', '.join(str(addr) for addr in ipv6_addresses[:3]) LOGGER.debug( f"Domain {check_domain} IPv6 AAAA records: " - f"{', '.join(ipv6_addresses[:3])}" + f"{ipv6_display}" ) + ipv6_display = ', '.join(str(addr) for addr in ipv6_addresses[:3]) LOGGER.info( f"Domain {check_domain} IPv6 AAAA records: " - f"{', '.join(ipv6_addresses[:3])}" + f"{ipv6_display}" ) # Check if any record matches the external IPs @@ -1000,14 +1093,16 @@ def check_domain_a_record(domain, external_ips=None): ipv6_match = False if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking IP address matches with server external IPs") + LOGGER.debug("Checking IP address matches with server " + "external IPs") # Check IPv4 match if external_ips.get("ipv4") and ipv4_addresses: if external_ips["ipv4"] in ipv4_addresses: + external_ipv4 = external_ips['ipv4'] LOGGER.info( f"✓ Domain {check_domain} IPv4 A record matches " - f"server external IP ({external_ips['ipv4']})" + f"server external IP ({external_ipv4})" ) ipv4_match = True else: @@ -1015,36 +1110,40 @@ def check_domain_a_record(domain, external_ips=None): f"⚠ Domain {check_domain} IPv4 A record does not " "match server external IP" ) - LOGGER.warning( - f" Domain IPv4: {', '.join(ipv4_addresses)}" - ) - LOGGER.warning(f" Server IPv4: {external_ips['ipv4']}") + ipv4_list = ', '.join(str(addr) for addr in ipv4_addresses) + LOGGER.warning(f" Domain IPv4: {ipv4_list}") + LOGGER.warning(f" Server IPv4: " + f"{external_ips['ipv4']}") # Check IPv6 match if external_ips.get("ipv6") and ipv6_addresses: if external_ips["ipv6"] in ipv6_addresses: + external_ipv6 = external_ips['ipv6'] LOGGER.info( - f"✓ Domain {check_domain} IPv6 AAAA record matches " - f"server external IP ({external_ips['ipv6']})" + f"✓ Domain {check_domain} IPv6 AAAA record " + f"matches server external IP ({external_ipv6})" ) ipv6_match = True else: LOGGER.warning( - f"⚠ Domain {check_domain} IPv6 AAAA record does not " - "match server external IP" - ) - LOGGER.warning( - f" Domain IPv6: {', '.join(ipv6_addresses)}" + f"⚠ Domain {check_domain} IPv6 AAAA record does " + "not match server external IP" ) - LOGGER.warning(f" Server IPv6: {external_ips['ipv6']}") + ipv6_list = ', '.join(str(addr) for addr in ipv6_addresses) + LOGGER.warning(f" Domain IPv6: {ipv6_list}") + LOGGER.warning(f" Server IPv6: " + f"{external_ips['ipv6']}") # Determine if we have any matching records has_any_match = ipv4_match or ipv6_match - has_external_ip = external_ips.get("ipv4") or external_ips.get("ipv6") + has_external_ip = (external_ips.get("ipv4") or + external_ips.get("ipv6")) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"IP match results: IPv4={ipv4_match}, IPv6={ipv6_match}") - LOGGER.debug(f"Has external IP: {has_external_ip}, Has match: {has_any_match}") + LOGGER.debug(f"IP match results: IPv4={ipv4_match}, " + f"IPv6={ipv6_match}") + LOGGER.debug(f"Has external IP: {has_external_ip}, " + f"Has match: {has_any_match}") if has_external_ip and not has_any_match: LOGGER.warning( @@ -1052,18 +1151,21 @@ def check_domain_a_record(domain, external_ips=None): "any server external IP" ) LOGGER.warning( - f" HTTP challenge may fail - ensure domain points to this server" + f" HTTP challenge may fail - ensure domain points " + f"to this server" ) # Check if we should treat this as an error - strict_ip_check = getenv("ACME_HTTP_STRICT_IP_CHECK", "no") == "yes" + strict_ip_check = (getenv("ACME_HTTP_STRICT_IP_CHECK", + "no") == "yes") if strict_ip_check: LOGGER.error( f"Strict IP check enabled - rejecting certificate " f"request for {check_domain}" ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Strict IP check failed - returning False") + LOGGER.debug("Strict IP check failed - " + "returning False") return False LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") @@ -1073,21 +1175,27 @@ def check_domain_a_record(domain, external_ips=None): else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug( - f"Domain {check_domain} validation failed - no DNS resolution" + f"Domain {check_domain} validation failed - no DNS " + f"resolution" ) - LOGGER.info(f"Domain {check_domain} validation failed - no DNS resolution") + LOGGER.info(f"Domain {check_domain} validation failed - no DNS " + f"resolution") LOGGER.warning(f"Domain {check_domain} does not resolve") return False except gaierror as e: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {check_domain} DNS resolution failed (gaierror): {e}") - LOGGER.info(f"Domain {check_domain} DNS resolution failed (gaierror): {e}") - LOGGER.warning(f"DNS resolution failed for domain {check_domain}: {e}") + LOGGER.debug(f"Domain {check_domain} DNS resolution failed " + f"(gaierror): {e}") + LOGGER.info(f"Domain {check_domain} DNS resolution failed " + f"(gaierror): {e}") + LOGGER.warning(f"DNS resolution failed for domain {check_domain}: " + f"{e}") return False except BaseException as e: LOGGER.info(format_exc()) - LOGGER.error(f"Error checking DNS records for domain {check_domain}: {e}") + LOGGER.error(f"Error checking DNS records for domain " + f"{check_domain}: {e}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("DNS check failed with unexpected exception") return False @@ -1097,40 +1205,40 @@ def certbot_new_with_retry( challenge_type: Literal["dns", "http"], domains: str, email: str, - provider: str = None, - credentials_path: Union[str, Path] = None, + provider: Optional[str] = None, + credentials_path: Optional[Union[str, Path]] = None, propagation: str = "default", profile: str = "classic", staging: bool = False, force: bool = False, - cmd_env: Dict[str, str] = None, + cmd_env: Optional[Dict[str, str]] = None, max_retries: int = 0, ca_provider: str = "letsencrypt", - api_key: str = None, - server_name: str = None, + api_key: Optional[str] = None, + server_name: Optional[str] = None, ) -> int: # Execute certbot with retry mechanism. # Wrapper around certbot_new that implements automatic retries with # exponential backoff for failed certificate generation attempts. # # Args: - # challenge_type - Type of ACME challenge ('dns' or 'http') - # domains - Comma-separated list of domain names - # email - Email address for certificate registration - # provider - DNS provider name (for DNS challenge) - # credentials_path - Path to credentials file - # propagation - DNS propagation time in seconds - # profile - Certificate profile to use - # staging - Whether to use staging environment - # force - Force renewal of existing certificates - # cmd_env - Environment variables for certbot process - # max_retries - Maximum number of retry attempts - # ca_provider - Certificate authority provider - # api_key - API key for CA (if required) - # server_name - Server name for multisite configurations + # challenge_type: Type of ACME challenge ('dns' or 'http') + # domains: Comma-separated list of domain names + # email: Email address for certificate registration + # provider: DNS provider name (for DNS challenge) + # credentials_path: Path to credentials file + # propagation: DNS propagation time in seconds + # profile: Certificate profile to use + # staging: Whether to use staging environment + # force: Force renewal of existing certificates + # cmd_env: Environment variables for certbot process + # max_retries: Maximum number of retry attempts + # ca_provider: Certificate authority provider + # api_key: API key for CA (if required) + # server_name: Server name for multisite configurations # # Returns: - # int - Exit code (0 for success) + # int: Exit code (0 for success) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Starting certbot with retry for domains: {domains}") LOGGER.debug(f"Max retries: {max_retries}, CA: {ca_provider}") @@ -1147,7 +1255,8 @@ def certbot_new_with_retry( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Waiting {wait_time} seconds before retry...") - LOGGER.debug(f"Exponential backoff: base=30s, attempt={attempt}") + LOGGER.debug(f"Exponential backoff: base=30s, " + f"attempt={attempt}") LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) @@ -1164,7 +1273,7 @@ def certbot_new_with_retry( profile, staging, force, - cmd_env, + cmd_env or {}, ca_provider, api_key, server_name, @@ -1172,13 +1281,15 @@ def certbot_new_with_retry( if result == 0: if attempt > 1: - LOGGER.info(f"Certificate generation succeeded on attempt {attempt}") + LOGGER.info(f"Certificate generation succeeded on attempt " + f"{attempt}") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Certbot completed successfully") return result if attempt >= max_retries + 1: - LOGGER.error(f"Certificate generation failed after {max_retries + 1} attempts") + LOGGER.error(f"Certificate generation failed after " + f"{max_retries + 1} attempts") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Maximum retries reached - giving up") return result @@ -1194,38 +1305,38 @@ def certbot_new( challenge_type: Literal["dns", "http"], domains: str, email: str, - provider: str = None, - credentials_path: Union[str, Path] = None, + provider: Optional[str] = None, + credentials_path: Optional[Union[str, Path]] = None, propagation: str = "default", profile: str = "classic", staging: bool = False, force: bool = False, - cmd_env: Dict[str, str] = None, + cmd_env: Optional[Dict[str, str]] = None, ca_provider: str = "letsencrypt", - api_key: str = None, - server_name: str = None, + api_key: Optional[str] = None, + server_name: Optional[str] = None, ) -> int: # Generate new certificate using certbot. # Main function to request SSL/TLS certificates from a certificate authority # using the ACME protocol via certbot. # # Args: - # challenge_type - Type of ACME challenge ('dns' or 'http') - # domains - Comma-separated list of domain names - # email - Email address for certificate registration - # provider - DNS provider name (for DNS challenge) - # credentials_path - Path to credentials file - # propagation - DNS propagation time in seconds - # profile - Certificate profile to use - # staging - Whether to use staging environment - # force - Force renewal of existing certificates - # cmd_env - Environment variables for certbot process - # ca_provider - Certificate authority provider - # api_key - API key for CA (if required) - # server_name - Server name for multisite configurations + # challenge_type: Type of ACME challenge ('dns' or 'http') + # domains: Comma-separated list of domain names + # email: Email address for certificate registration + # provider: DNS provider name (for DNS challenge) + # credentials_path: Path to credentials file + # propagation: DNS propagation time in seconds + # profile: Certificate profile to use + # staging: Whether to use staging environment + # force: Force renewal of existing certificates + # cmd_env: Environment variables for certbot process + # ca_provider: Certificate authority provider + # api_key: API key for CA (if required) + # server_name: Server name for multisite configurations # # Returns: - # int - Exit code (0 for success) + # int: Exit code (0 for success) if isinstance(credentials_path, str): credentials_path = Path(credentials_path) @@ -1241,7 +1352,7 @@ def certbot_new( CERTBOT_BIN, "certonly", "--config-dir", - DATA_PATH.as_posix(), + str(DATA_PATH), "--work-dir", WORK_DIR, "--logs-dir", @@ -1258,8 +1369,13 @@ def certbot_new( ca_config["server"], ] - if not cmd_env: + # Ensure we have a valid environment dictionary to work with + if cmd_env is None: cmd_env = {} + + # Create a properly typed working environment dictionary + working_env: Dict[str, str] = {} + working_env.update(cmd_env) # Copy existing values if any # Handle certificate key type based on DNS provider and CA if challenge_type == "dns" and provider in ("infomaniak", "ionos"): @@ -1267,7 +1383,8 @@ def certbot_new( command.extend(["--rsa-key-size", "4096"]) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using RSA-4096 for {provider} provider with {domains}") + LOGGER.debug(f"Using RSA-4096 for {provider} provider with " + f"{domains}") LOGGER.info(f"Using RSA-4096 for {provider} provider with {domains}") else: # Use elliptic curve certificates for all other providers @@ -1294,9 +1411,11 @@ def certbot_new( # Check for manually provided EAB credentials first eab_kid_env = (getenv("ACME_ZEROSSL_EAB_KID", "") or - getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "")) + (getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "") + if server_name else "")) eab_hmac_env = (getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or - getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "")) + (getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") + if server_name else "")) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Manual EAB credentials check:") @@ -1304,17 +1423,22 @@ def certbot_new( LOGGER.debug(f" EAB HMAC provided: {bool(eab_hmac_env)}") if eab_kid_env and eab_hmac_env: - LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials from environment") - command.extend(["--eab-kid", eab_kid_env, "--eab-hmac-key", eab_hmac_env]) + LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials " + "from environment") + command.extend(["--eab-kid", eab_kid_env, "--eab-hmac-key", + eab_hmac_env]) LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid_env[:10]}...") elif api_key: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL API key provided, setting up EAB credentials") - LOGGER.info(f"ZeroSSL API key provided, setting up EAB credentials") + LOGGER.debug(f"ZeroSSL API key provided, setting up EAB " + f"credentials") + LOGGER.info(f"ZeroSSL API key provided, setting up EAB " + f"credentials") eab_kid, eab_hmac = setup_zerossl_eab_credentials(email, api_key) if eab_kid and eab_hmac: - command.extend(["--eab-kid", eab_kid, "--eab-hmac-key", eab_hmac]) + command.extend(["--eab-kid", eab_kid, "--eab-hmac-key", + eab_hmac]) LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid[:10]}...") else: @@ -1323,12 +1447,14 @@ def certbot_new( "Alternative: Set ACME_ZEROSSL_EAB_KID and " "ACME_ZEROSSL_EAB_HMAC_KEY environment variables" ) - LOGGER.warning("Proceeding without EAB - this will likely fail") + LOGGER.warning("Proceeding without EAB - this will likely " + "fail") else: LOGGER.error("❌ No ZeroSSL API key provided!") LOGGER.error("Set ACME_ZEROSSL_API_KEY environment variable") LOGGER.error( - "Or set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY directly" + "Or set ACME_ZEROSSL_EAB_KID and ACME_ZEROSSL_EAB_HMAC_KEY " + "directly" ) LOGGER.warning("Proceeding without EAB - this will likely fail") @@ -1348,19 +1474,29 @@ def certbot_new( "using provider's default..." ) else: - command.extend([f"--dns-{provider}-propagation-seconds", propagation]) + command.extend([f"--dns-{provider}-propagation-seconds", + propagation]) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Set DNS propagation time to {propagation} seconds") + LOGGER.debug(f"Set DNS propagation time to " + f"{propagation} seconds") if provider == "route53": if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Route53 provider - setting environment variables") - with credentials_path.open("r") as file: - for line in file: - key, value = line.strip().split("=", 1) - cmd_env[key] = value + LOGGER.debug("Route53 provider - setting environment " + "variables") + if credentials_path: + with open(credentials_path, "r") as file: + for line in file: + if '=' in line: + key, value = line.strip().split("=", 1) + # Explicit type-safe assignment + env_key: str = str(key) + env_value: str = str(value) + working_env[env_key] = env_value else: - command.extend([f"--dns-{provider}-credentials", credentials_path.as_posix()]) + if credentials_path: + command.extend([f"--dns-{provider}-credentials", + str(credentials_path)]) if provider in ("desec", "infomaniak", "ionos", "njalla", "scaleway"): command.extend(["--authenticator", f"dns-{provider}"]) @@ -1371,18 +1507,20 @@ def certbot_new( elif challenge_type == "http": if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": + auth_hook = JOBS_PATH.joinpath('certbot-auth.py') + cleanup_hook = JOBS_PATH.joinpath('certbot-cleanup.py') LOGGER.debug("HTTP challenge configuration:") - LOGGER.debug(f" Auth hook: {JOBS_PATH.joinpath('certbot-auth.py')}") - LOGGER.debug(f" Cleanup hook: {JOBS_PATH.joinpath('certbot-cleanup.py')}") + LOGGER.debug(f" Auth hook: {auth_hook}") + LOGGER.debug(f" Cleanup hook: {cleanup_hook}") command.extend( [ "--manual", "--preferred-challenges=http", "--manual-auth-hook", - JOBS_PATH.joinpath("certbot-auth.py").as_posix(), + str(JOBS_PATH.joinpath("certbot-auth.py")), "--manual-cleanup-hook", - JOBS_PATH.joinpath("certbot-cleanup.py").as_posix(), + str(JOBS_PATH.joinpath("certbot-cleanup.py")), ] ) @@ -1412,17 +1550,20 @@ def certbot_new( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Command: {' '.join(safe_command)}") - LOGGER.debug(f"Environment variables: {len(cmd_env)} items") - for key in cmd_env.keys(): - LOGGER.debug(f" {key}: {'***MASKED***' if 'key' in key.lower() or 'secret' in key.lower() or 'token' in key.lower() else 'set'}") + LOGGER.debug(f"Environment variables: {len(working_env)} items") + for key in working_env.keys(): + is_sensitive = any(sensitive in key.lower() + for sensitive in ['key', 'secret', 'token']) + value_display = '***MASKED***' if is_sensitive else 'set' + LOGGER.debug(f" {key}: {value_display}") LOGGER.info(f"Command: {' '.join(safe_command)}") current_date = datetime.now() if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Starting certbot process") - process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, - env=cmd_env) + process = Popen(command, stdin=DEVNULL, stderr=PIPE, + universal_newlines=True, env=working_env) lines_processed = 0 while process.poll() is None: @@ -1446,11 +1587,13 @@ def certbot_new( current_date = datetime.now() if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot still running, processed {lines_processed} output lines") + LOGGER.debug(f"Certbot still running, processed " + f"{lines_processed} output lines") final_return_code = process.returncode if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot process completed with return code: {final_return_code}") + LOGGER.debug(f"Certbot process completed with return code: " + f"{final_return_code}") LOGGER.debug(f"Total output lines processed: {lines_processed}") return final_return_code @@ -1492,21 +1635,27 @@ def certbot_new( domains_server_names = {} for first_server in servers: - if first_server and getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes": + auto_le_env = f"{first_server}_AUTO_LETS_ENCRYPT" + if (first_server and + getenv(auto_le_env, "no") == "yes"): use_letsencrypt = True + challenge_env = f"{first_server}_LETS_ENCRYPT_CHALLENGE" if (first_server and - getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") == "dns"): + getenv(challenge_env, "http") == "dns"): use_letsencrypt_dns = True + server_name_env = f"{first_server}_SERVER_NAME" domains_server_names[first_server] = getenv( - f"{first_server}_SERVER_NAME", first_server + server_name_env, first_server ).lower() if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Multi-site configuration:") - LOGGER.debug(f" Let's Encrypt enabled anywhere: {use_letsencrypt}") - LOGGER.debug(f" DNS challenge used anywhere: {use_letsencrypt_dns}") + LOGGER.debug(f" Let's Encrypt enabled anywhere: " + f"{use_letsencrypt}") + LOGGER.debug(f" DNS challenge used anywhere: " + f"{use_letsencrypt_dns}") LOGGER.debug(f" Domain mappings: {domains_server_names}") if not use_letsencrypt: @@ -1569,19 +1718,25 @@ def certbot_new( LOGGER.debug("Restoring certificate data from database cache") JOB.restore_cache(job_name="certbot-renew") - env = { - "PATH": getenv("PATH", ""), - "PYTHONPATH": getenv("PYTHONPATH", ""), - "RELOAD_MIN_TIMEOUT": getenv("RELOAD_MIN_TIMEOUT", "5"), - "DISABLE_CONFIGURATION_TESTING": getenv( - "DISABLE_CONFIGURATION_TESTING", "no" + # Initialize environment variables for certbot execution + env: Dict[str, str] = { + "PATH": getenv("PATH") or "", + "PYTHONPATH": getenv("PYTHONPATH") or "", + "RELOAD_MIN_TIMEOUT": getenv("RELOAD_MIN_TIMEOUT") or "5", + "DISABLE_CONFIGURATION_TESTING": ( + getenv("DISABLE_CONFIGURATION_TESTING") or "no" ).lower(), } + env["PYTHONPATH"] = env["PYTHONPATH"] + ( f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" ) - if getenv("DATABASE_URI"): - env["DATABASE_URI"] = getenv("DATABASE_URI") + database_uri = getenv("DATABASE_URI") + if database_uri: # Only assign if not None and not empty + # Explicit assignment with type safety + env_key: str = "DATABASE_URI" + env_value: str = str(database_uri) + env[env_key] = env_value if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Checking existing certificates") @@ -1591,7 +1746,7 @@ def certbot_new( CERTBOT_BIN, "certificates", "--config-dir", - DATA_PATH.as_posix(), + str(DATA_PATH), "--work-dir", WORK_DIR, "--logs-dir", @@ -1618,35 +1773,41 @@ def certbot_new( LOGGER.debug("Certificate listing failed - proceeding anyway") else: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate listing successful - analyzing existing certificates") + LOGGER.debug("Certificate listing successful - analyzing " + "existing certificates") certificate_blocks = stdout.split("Certificate Name: ")[1:] if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found {len(certificate_blocks)} existing certificates") + LOGGER.debug(f"Found {len(certificate_blocks)} existing " + f"certificates") for first_server, domains in domains_server_names.items(): - if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): + auto_le_check = (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") + if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) + if auto_le_check != "yes": continue - letsencrypt_challenge = ( - getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") if IS_MULTISITE + challenge_check = ( + getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") + if IS_MULTISITE else getenv("LETS_ENCRYPT_CHALLENGE", "http") ) original_first_server = deepcopy(first_server) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Processing server: {first_server}") - LOGGER.debug(f" Challenge: {letsencrypt_challenge}") + LOGGER.debug(f" Challenge: {challenge_check}") LOGGER.debug(f" Domains: {domains}") - if ( - letsencrypt_challenge == "dns" - and (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_WILDCARD", "no")) == "yes" - ): + wildcard_check = ( + getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", + "no") + if IS_MULTISITE + else getenv("USE_LETS_ENCRYPT_WILDCARD", "no") + ) + if (challenge_check == "dns" and wildcard_check == "yes"): if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Using wildcard mode for {first_server}") @@ -1656,7 +1817,7 @@ def certbot_new( first_server = wildcards[0].lstrip("*.") domains = set(wildcards) else: - domains = set(domains.split(" ")) + domains = set(str(domains).split(" ")) # Add the certificate name to our active set regardless # if we're generating it or not @@ -1703,29 +1864,33 @@ def certbot_new( cert_domains_set = set(cert_domains_list) desired_domains_set = ( set(domains) if isinstance(domains, (list, set)) - else set(domains.split()) + else set(str(domains).split()) ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate domain comparison for {first_server}:") - LOGGER.debug(f" Existing: {sorted(cert_domains_set)}") - LOGGER.debug(f" Desired: {sorted(desired_domains_set)}") + LOGGER.debug(f"Certificate domain comparison for " + f"{first_server}:") + LOGGER.debug(f" Existing: {sorted(str(d) for d in cert_domains_set)}") + LOGGER.debug(f" Desired: {sorted(str(d) for d in desired_domains_set)}") if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 + existing_sorted = sorted(str(d) for d in cert_domains_set) + desired_sorted = sorted(str(d) for d in desired_domains_set) LOGGER.warning( - f"[{original_first_server}] Domains for {first_server} differ " - f"from desired set (existing: {sorted(cert_domains_set)}, " - f"desired: {sorted(desired_domains_set)}), asking new certificate..." + f"[{original_first_server}] Domains for {first_server} " + f"differ from desired set (existing: {existing_sorted}, " + f"desired: {desired_sorted}), asking new certificate..." ) continue # Check if CA provider has changed - ca_provider = ( - getenv(f"{original_first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") + ca_provider_env = ( + f"{original_first_server}_ACME_SSL_CA_PROVIDER" if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") + else "ACME_SSL_CA_PROVIDER" ) + ca_provider = getenv(ca_provider_env, "letsencrypt") renewal_file = DATA_PATH.joinpath("renewal", f"{first_server}.conf") if renewal_file.is_file(): @@ -1736,11 +1901,14 @@ def certbot_new( current_server = line.strip().split("=", 1)[1].strip() break + staging_env = ( + f"{original_first_server}_USE_LETS_ENCRYPT_STAGING" + if IS_MULTISITE + else "USE_LETS_ENCRYPT_STAGING" + ) + staging_mode = getenv(staging_env, "no") == "yes" expected_config = get_certificate_authority_config( - ca_provider, - (getenv(f"{original_first_server}_USE_LETS_ENCRYPT_STAGING", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_STAGING", "no")) == "yes" + ca_provider, staging_mode ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -1748,19 +1916,22 @@ def certbot_new( LOGGER.debug(f" Current: {current_server}") LOGGER.debug(f" Expected: {expected_config['server']}") - if current_server and current_server != expected_config["server"]: + if (current_server and + current_server != expected_config["server"]): domains_to_ask[first_server] = 2 LOGGER.warning( f"[{original_first_server}] CA provider for " - f"{first_server} has changed, asking new certificate..." + f"{first_server} has changed, asking new " + f"certificate..." ) continue - use_staging = ( - getenv(f"{original_first_server}_USE_LETS_ENCRYPT_STAGING", "no") + staging_env = ( + f"{original_first_server}_USE_LETS_ENCRYPT_STAGING" if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_STAGING", "no") - ) == "yes" + else "USE_LETS_ENCRYPT_STAGING" + ) + use_staging = getenv(staging_env, "no") == "yes" is_test_cert = "TEST_CERT" in cert_domains.group("expiry_date") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -1768,7 +1939,9 @@ def certbot_new( LOGGER.debug(f" Use staging: {use_staging}") LOGGER.debug(f" Is test cert: {is_test_cert}") - if (is_test_cert and not use_staging) or (not is_test_cert and use_staging): + staging_mismatch = ((is_test_cert and not use_staging) or + (not is_test_cert and use_staging)) + if staging_mismatch: domains_to_ask[first_server] = 2 LOGGER.warning( f"[{original_first_server}] Certificate environment " @@ -1777,11 +1950,12 @@ def certbot_new( ) continue - provider = ( - getenv(f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") + provider_env = ( + f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER" if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROVIDER", "") + else "LETS_ENCRYPT_DNS_PROVIDER" ) + provider = getenv(provider_env, "") if not renewal_file.is_file(): LOGGER.error( @@ -1804,7 +1978,7 @@ def certbot_new( LOGGER.debug(f" Current: {current_provider}") LOGGER.debug(f" Configured: {provider}") - if letsencrypt_challenge == "dns": + if challenge_check == "dns": if provider and current_provider != provider: domains_to_ask[first_server] = 2 LOGGER.warning( @@ -1817,7 +1991,8 @@ def certbot_new( # Check if DNS credentials have changed if provider and current_provider == provider: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking DNS credentials for {first_server}") + LOGGER.debug(f"Checking DNS credentials for " + f"{first_server}") credential_key = ( f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" @@ -1832,51 +2007,63 @@ def certbot_new( current_credential_items["json_data"] = env_value continue key, value = env_value.split(" ", 1) - current_credential_items[key.lower()] = ( + cleaned_value = ( value.removeprefix("= ").replace("\\n", "\n") - .replace("\\t", "\t").replace("\\r", "\r").strip() + .replace("\\t", "\t").replace("\\r", "\r") + .strip() ) + current_credential_items[key.lower()] = cleaned_value if "json_data" in current_credential_items: value = current_credential_items.pop("json_data") - if (not current_credential_items and len(value) % 4 == 0 - and match(r"^[A-Za-z0-9+/=]+$", value)): + is_base64_like = (not current_credential_items and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value)) + if is_base64_like: with suppress(BaseException): decoded = b64decode(value).decode("utf-8") json_data = loads(decoded) if isinstance(json_data, dict): - current_credential_items = { - k.lower(): str(v).removeprefix("= ") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\r", "\r").strip() - for k, v in json_data.items() - } + new_items = {} + for k, v in json_data.items(): + cleaned_v = ( + str(v).removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r") + .strip() + ) + new_items[k.lower()] = cleaned_v + current_credential_items = new_items if current_credential_items: for key, value in current_credential_items.items(): - if (provider != "rfc2136" and len(value) % 4 == 0 - and match(r"^[A-Za-z0-9+/=]+$", value)): + is_base64_candidate = (provider != "rfc2136" and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value)) + if is_base64_candidate: with suppress(BaseException): decoded = b64decode(value).decode("utf-8") if decoded != value: - current_credential_items[key] = ( + cleaned_decoded = ( decoded.removeprefix("= ") .replace("\\n", "\n") .replace("\\t", "\t") - .replace("\\r", "\r").strip() + .replace("\\r", "\r") + .strip() ) + current_credential_items[key] = cleaned_decoded if provider in provider_classes: with suppress(ValidationError, KeyError): - current_provider_instance = provider_classes[provider]( + provider_instance = provider_classes[provider]( **current_credential_items ) current_credentials_content = ( - current_provider_instance.get_formatted_credentials() + provider_instance.get_formatted_credentials() ) - file_type = current_provider_instance.get_file_type() + file_type = provider_instance.get_file_type() stored_credentials_path = CACHE_PATH.joinpath( first_server, f"credentials.{file_type}" ) @@ -1885,16 +2072,21 @@ def certbot_new( stored_credentials_content = ( stored_credentials_path.read_bytes() ) - if (stored_credentials_content != - current_credentials_content): + content_differs = ( + stored_credentials_content != + current_credentials_content + ) + if content_differs: domains_to_ask[first_server] = 2 LOGGER.warning( - f"[{original_first_server}] DNS credentials " - f"for {first_server} have changed, " - "asking new certificate..." + f"[{original_first_server}] DNS " + f"credentials for {first_server} " + f"have changed, asking new " + f"certificate..." ) continue - elif current_provider != "manual" and letsencrypt_challenge == "http": + elif (current_provider != "manual" and + challenge_check == "http"): domains_to_ask[first_server] = 2 LOGGER.warning( f"[{original_first_server}] {first_server} is no longer " @@ -1917,8 +2109,10 @@ def certbot_new( # Process each server configuration for first_server, domains in domains_server_names.items(): - if ((getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) != "yes"): + auto_le_check = (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") + if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no")) + if auto_le_check != "yes": LOGGER.info( f"SSL certificate generation is not activated for " f"{first_server}, skipping..." @@ -1926,60 +2120,50 @@ def certbot_new( continue # Getting all the necessary data + email_env = (f"{first_server}_EMAIL_LETS_ENCRYPT" if IS_MULTISITE + else "EMAIL_LETS_ENCRYPT") + challenge_env = (f"{first_server}_LETS_ENCRYPT_CHALLENGE" + if IS_MULTISITE + else "LETS_ENCRYPT_CHALLENGE") + staging_env = (f"{first_server}_USE_LETS_ENCRYPT_STAGING" + if IS_MULTISITE + else "USE_LETS_ENCRYPT_STAGING") + wildcard_env = (f"{first_server}_USE_LETS_ENCRYPT_WILDCARD" + if IS_MULTISITE + else "USE_LETS_ENCRYPT_WILDCARD") + provider_env = (f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_PROVIDER") + propagation_env = (f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_PROPAGATION") + profile_env = (f"{first_server}_LETS_ENCRYPT_PROFILE" + if IS_MULTISITE + else "LETS_ENCRYPT_PROFILE") + psl_env = (f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES" + if IS_MULTISITE + else "LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES") + retries_env = (f"{first_server}_LETS_ENCRYPT_MAX_RETRIES" + if IS_MULTISITE + else "LETS_ENCRYPT_MAX_RETRIES") + ca_env = (f"{first_server}_ACME_SSL_CA_PROVIDER" if IS_MULTISITE + else "ACME_SSL_CA_PROVIDER") + api_key_env = (f"{first_server}_ACME_ZEROSSL_API_KEY" + if IS_MULTISITE + else "ACME_ZEROSSL_API_KEY") + data = { - "email": ( - getenv(f"{first_server}_EMAIL_LETS_ENCRYPT", "") if IS_MULTISITE - else getenv("EMAIL_LETS_ENCRYPT", "") - ) or f"contact@{first_server}", - "challenge": ( - getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_CHALLENGE", "http") - ), - "staging": ( - getenv(f"{first_server}_USE_LETS_ENCRYPT_STAGING", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_STAGING", "no") - ) == "yes", - "use_wildcard": ( - getenv(f"{first_server}_USE_LETS_ENCRYPT_WILDCARD", "no") - if IS_MULTISITE - else getenv("USE_LETS_ENCRYPT_WILDCARD", "no") - ) == "yes", - "provider": ( - getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER", "") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROVIDER", "") - ), - "propagation": ( - getenv(f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION", "default") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_DNS_PROPAGATION", "default") - ), - "profile": ( - getenv(f"{first_server}_LETS_ENCRYPT_PROFILE", "classic") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_PROFILE", "classic") - ), - "check_psl": ( - getenv(f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES", "yes") - ) == "no", - "max_retries": ( - getenv(f"{first_server}_LETS_ENCRYPT_MAX_RETRIES", "0") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_MAX_RETRIES", "0") - ), - "ca_provider": ( - getenv(f"{first_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") - if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") - ), - "api_key": ( - getenv(f"{first_server}_ACME_ZEROSSL_API_KEY", "") if IS_MULTISITE - else getenv("ACME_ZEROSSL_API_KEY", "") - ), + "email": (getenv(email_env, "") or f"contact@{first_server}"), + "challenge": getenv(challenge_env, "http"), + "staging": getenv(staging_env, "no") == "yes", + "use_wildcard": getenv(wildcard_env, "no") == "yes", + "provider": getenv(provider_env, ""), + "propagation": getenv(propagation_env, "default"), + "profile": getenv(profile_env, "classic"), + "check_psl": getenv(psl_env, "yes") == "no", + "max_retries": getenv(retries_env, "0"), + "ca_provider": getenv(ca_env, "letsencrypt"), + "api_key": getenv(api_key_env, ""), "credential_items": {}, } @@ -1988,17 +2172,17 @@ def certbot_new( LOGGER.info(f"Service {first_server} configuration:") LOGGER.info(f" CA Provider: {data['ca_provider']}") - LOGGER.info(f" API Key provided: {'Yes' if data['api_key'] else 'No'}") + api_key_status = 'Yes' if data['api_key'] else 'No' + LOGGER.info(f" API Key provided: {api_key_status}") LOGGER.info(f" Challenge type: {data['challenge']}") LOGGER.info(f" Staging: {data['staging']}") LOGGER.info(f" Wildcard: {data['use_wildcard']}") # Override profile if custom profile is set - custom_profile = ( - getenv(f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE", "") - if IS_MULTISITE - else getenv("LETS_ENCRYPT_CUSTOM_PROFILE", "") - ).strip() + custom_profile_env = (f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE" + if IS_MULTISITE + else "LETS_ENCRYPT_CUSTOM_PROFILE") + custom_profile = getenv(custom_profile_env, "").strip() if custom_profile: data["profile"] = custom_profile if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -2011,12 +2195,16 @@ def certbot_new( ) data["use_wildcard"] = False - if (not data["use_wildcard"] and not domains_to_ask.get(first_server)) or ( - data["use_wildcard"] and not domains_to_ask.get( - WILDCARD_GENERATOR.extract_wildcards_from_domains((first_server,))[0] - .lstrip("*.") - ) - ): + should_skip_cert_check = ( + (not data["use_wildcard"] and + not domains_to_ask.get(first_server)) or + (data["use_wildcard"] and not domains_to_ask.get( + WILDCARD_GENERATOR.extract_wildcards_from_domains( + (first_server,) + )[0].lstrip("*.") + )) + ) + if should_skip_cert_check: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"No certificate needed for {first_server}") continue @@ -2049,29 +2237,36 @@ def certbot_new( credential_items["json_data"] = env_value continue key, value = env_value.split(" ", 1) - credential_items[key.lower()] = ( + cleaned_value = ( value.removeprefix("= ") .replace("\\n", "\n") .replace("\\t", "\t") .replace("\\r", "\r").strip() ) + credential_items[key.lower()] = cleaned_value if "json_data" in credential_items: value = credential_items.pop("json_data") - # Handle the case of a single credential that might be base64-encoded JSON - if (not credential_items and len(value) % 4 == 0 - and match(r"^[A-Za-z0-9+/=]+$", value)): + # Handle the case of a single credential that might be + # base64-encoded JSON + is_potential_json = (not credential_items and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value)) + if is_potential_json: try: decoded = b64decode(value).decode("utf-8") json_data = loads(decoded) if isinstance(json_data, dict): - data["credential_items"] = { - k.lower(): str(v).removeprefix("= ") - .replace("\\n", "\n") - .replace("\\t", "\t") - .replace("\\r", "\r").strip() - for k, v in json_data.items() - } + new_items = {} + for k, v in json_data.items(): + cleaned_v = ( + str(v).removeprefix("= ") + .replace("\\n", "\n") + .replace("\\t", "\t") + .replace("\\r", "\r").strip() + ) + new_items[k.lower()] = cleaned_v + data["credential_items"] = new_items except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error( @@ -2084,8 +2279,10 @@ def certbot_new( data["credential_items"] = {} for key, value in credential_items.items(): # Check for base64 encoding - if (data["provider"] != "rfc2136" and len(value) % 4 == 0 - and match(r"^[A-Za-z0-9+/=]+$", value)): + is_base64_candidate = (data["provider"] != "rfc2136" and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value)) + if is_base64_candidate: try: decoded = b64decode(value).decode("utf-8") if decoded != value: @@ -2105,52 +2302,63 @@ def certbot_new( if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": safe_data = data.copy() - safe_data["credential_items"] = {k: "***MASKED***" for k in data["credential_items"].keys()} + masked_items = {k: "***MASKED***" + for k in data["credential_items"].keys()} + safe_data["credential_items"] = masked_items if data["api_key"]: safe_data["api_key"] = "***MASKED***" - LOGGER.debug(f"Safe data for service {first_server}: {dumps(safe_data)}") + LOGGER.debug(f"Safe data for service {first_server}: " + f"{dumps(safe_data)}") # Validate CA provider and API key requirements + api_key_status = 'Yes' if data['api_key'] else 'No' LOGGER.info( f"Service {first_server} - CA Provider: {data['ca_provider']}, " - f"API Key provided: {'Yes' if data['api_key'] else 'No'}" + f"API Key provided: {api_key_status}" ) if data["ca_provider"].lower() == "zerossl": if not data["api_key"]: LOGGER.warning( - f"ZeroSSL API key not provided for service {first_server}, " - "falling back to Let's Encrypt..." + f"ZeroSSL API key not provided for service " + f"{first_server}, falling back to Let's Encrypt..." ) data["ca_provider"] = "letsencrypt" else: - LOGGER.info(f"✓ ZeroSSL configuration valid for service {first_server}") + LOGGER.info(f"✓ ZeroSSL configuration valid for service " + f"{first_server}") # Checking if the DNS data is valid if data["challenge"] == "dns": if not data["provider"]: + available_providers = ', '.join(str(p) for p in provider_classes.keys()) LOGGER.warning( f"No provider found for service {first_server} " - f"(available providers: {', '.join(provider_classes.keys())}), " + f"(available providers: {available_providers}), " "skipping certificate(s) generation..." ) continue elif data["provider"] not in provider_classes: + available_providers = ', '.join(str(p) for p in provider_classes.keys()) LOGGER.warning( - f"Provider {data['provider']} not found for service {first_server} " - f"(available providers: {', '.join(provider_classes.keys())}), " - "skipping certificate(s) generation..." + f"Provider {data['provider']} not found for service " + f"{first_server} (available providers: " + f"{available_providers}), skipping certificate(s) " + f"generation..." ) continue elif not data["credential_items"]: LOGGER.warning( - f"No valid credentials items found for service {first_server} " - "(you should have at least one), skipping certificate(s) generation..." + f"No valid credentials items found for service " + f"{first_server} (you should have at least one), " + f"skipping certificate(s) generation..." ) continue try: - provider = provider_classes[data["provider"]](**data["credential_items"]) + provider = provider_classes[data["provider"]]( + **data["credential_items"] + ) except ValidationError as ve: LOGGER.debug(format_exc()) LOGGER.error( @@ -2166,31 +2374,37 @@ def certbot_new( is_blacklisted = False # Adding the domains to Wildcard Generator if necessary - file_type = (provider.get_file_type() if data["challenge"] == "dns" else "txt") + file_type = (provider.get_file_type() if data["challenge"] == "dns" + else "txt") file_path = (first_server, f"credentials.{file_type}") if data["use_wildcard"]: # Use the improved method for generating consistent group names group = WILDCARD_GENERATOR.create_group_name( domain=first_server, - provider=data["provider"] if data["challenge"] == "dns" else "http", + provider=(data["provider"] if data["challenge"] == "dns" + else "http"), challenge_type=data["challenge"], staging=data["staging"], content_hash=bytes_hash(content, algorithm="sha1"), profile=data["profile"], ) + wildcard_info = ( + "the propagation time will be the provider's default and " + if data["challenge"] == "dns" else "" + ) LOGGER.info( f"Service {first_server} is using wildcard, " - + ("the propagation time will be the provider's default and " - if data["challenge"] == "dns" else "") - + "the email will be the same as the first domain that created the group..." + f"{wildcard_info}the email will be the same as the first " + f"domain that created the group..." ) if data["check_psl"]: if psl_lines is None: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Loading PSL for wildcard domain validation") + LOGGER.debug("Loading PSL for wildcard domain " + "validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -2198,10 +2412,11 @@ def certbot_new( psl_rules = parse_psl(psl_lines) wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( - domains.split(" ") + str(domains).split(" ") ) - LOGGER.info(f"Wildcard domains for {first_server}: {wildcards}") + LOGGER.info(f"Wildcard domains for {first_server}: " + f"{wildcards}") for d in wildcards: if is_domain_blacklisted(d, psl_rules): @@ -2215,7 +2430,8 @@ def certbot_new( if not is_blacklisted: WILDCARD_GENERATOR.extend( - group, domains.split(" "), data["email"], data["staging"] + group, str(domains).split(" "), data["email"], + data["staging"] ) file_path = (f"{group}.{file_type}",) LOGGER.info(f"[{first_server}] Wildcard group {group}") @@ -2229,7 +2445,7 @@ def certbot_new( LOGGER.debug("Parsing PSL rules") psl_rules = parse_psl(psl_lines) - for d in domains.split(): + for d in str(domains).split(): if is_domain_blacklisted(d, psl_rules): LOGGER.error( f"Domain {d} is blacklisted by Public Suffix List, " @@ -2248,12 +2464,15 @@ def certbot_new( if data["challenge"] == "dns": if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Managing credentials file for {first_server}: {credentials_path}") + LOGGER.debug(f"Managing credentials file for {first_server}: " + f"{credentials_path}") if not credentials_path.is_file(): + service_id = (first_server if not data["use_wildcard"] + else "") cached, err = JOB.cache_file( credentials_path.name, content, job_name="certbot-renew", - service_id=first_server if not data["use_wildcard"] else "" + service_id=service_id ) if not cached: LOGGER.error( @@ -2294,55 +2513,61 @@ def certbot_new( ) else: LOGGER.info( - f"Service {first_server}'s credentials file is up to date" + f"Service {first_server}'s credentials file is " + f"up to date" ) credential_paths.add(credentials_path) - # Setting the permissions to 600 (this is important to avoid warnings from certbot) + # Setting the permissions to 600 (this is important to avoid + # warnings from certbot) credentials_path.chmod(0o600) if data["use_wildcard"]: if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Wildcard processing complete for {first_server}") + LOGGER.debug(f"Wildcard processing complete for " + f"{first_server}") continue - domains = domains.replace(" ", ",") + domains = str(domains).replace(" ", ",") ca_name = get_certificate_authority_config(data["ca_provider"])["name"] + staging_info = ' using staging' if data['staging'] else '' LOGGER.info( f"Asking {ca_name} certificates for domain(s): {domains} " - f"(email = {data['email']}){' using staging' if data['staging'] else ''} " - f" with {data['challenge']} challenge, using {data['profile']!r} profile..." + f"(email = {data['email']}){staging_info} " + f" with {data['challenge']} challenge, using " + f"{data['profile']!r} profile..." ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug(f"Requesting certificate for {domains}") - if ( - certbot_new_with_retry( - data["challenge"], - domains, - data["email"], - data["provider"], - credentials_path, - data["propagation"], - data["profile"], - data["staging"], - domains_to_ask[first_server] == 2, - cmd_env=env, - max_retries=data["max_retries"], - ca_provider=data["ca_provider"], - api_key=data["api_key"], - server_name=first_server, - ) - != 0 - ): + cert_result = certbot_new_with_retry( + data["challenge"], + domains, + data["email"], + data["provider"], + credentials_path, + data["propagation"], + data["profile"], + data["staging"], + domains_to_ask[first_server] == 2, + cmd_env=env, + max_retries=data["max_retries"], + ca_provider=data["ca_provider"], + api_key=data["api_key"], + server_name=first_server, + ) + + if cert_result != 0: status = 2 certificates_failed += 1 - LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") + LOGGER.error(f"Certificate generation failed for domain(s) " + f"{domains}...") else: status = 1 if status == 0 else status certificates_generated += 1 - LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") + LOGGER.info(f"Certificate generation succeeded for domain(s): " + f"{domains}") generated_domains.update(domains.split(",")) @@ -2369,9 +2594,9 @@ def certbot_new( LOGGER.debug(f" Base domain: {base_domain}") email = data.pop("email") - credentials_file = CACHE_PATH.joinpath( - f"{group}.{provider_classes[provider].get_file_type() if provider in provider_classes else 'txt'}" - ) + file_type = (provider_classes[provider].get_file_type() + if provider in provider_classes else 'txt') + credentials_file = CACHE_PATH.joinpath(f"{group}.{file_type}") # Get CA provider for this group original_server = None @@ -2383,16 +2608,15 @@ def certbot_new( ca_provider = "letsencrypt" # default api_key = None if original_server: - ca_provider = ( - getenv(f"{original_server}_ACME_SSL_CA_PROVIDER", "letsencrypt") - if IS_MULTISITE - else getenv("ACME_SSL_CA_PROVIDER", "letsencrypt") - ) - api_key = ( - getenv(f"{original_server}_ACME_ZEROSSL_API_KEY", "") - if IS_MULTISITE - else getenv("ACME_ZEROSSL_API_KEY", "") - ) + ca_env = (f"{original_server}_ACME_SSL_CA_PROVIDER" + if IS_MULTISITE + else "ACME_SSL_CA_PROVIDER") + ca_provider = getenv(ca_env, "letsencrypt") + + api_key_env = (f"{original_server}_ACME_ZEROSSL_API_KEY" + if IS_MULTISITE + else "ACME_ZEROSSL_API_KEY") + api_key = getenv(api_key_env, "") # Process different environment types (staging/prod) for key, domains in data.items(): @@ -2401,10 +2625,12 @@ def certbot_new( staging = key == "staging" ca_name = get_certificate_authority_config(ca_provider)["name"] + staging_info = ' using staging ' if staging else '' + challenge_type = 'dns' if provider in provider_classes else 'http' LOGGER.info( - f"Asking {ca_name} wildcard certificates for domain(s): {domains} " - f"(email = {email}){' using staging ' if staging else ''} " - f"with {'dns' if provider in provider_classes else 'http'} challenge, " + f"Asking {ca_name} wildcard certificates for domain(s): " + f"{domains} (email = {email}){staging_info} " + f"with {challenge_type} challenge, " f"using {profile!r} profile..." ) @@ -2417,38 +2643,41 @@ def certbot_new( active_cert_names.add(base_domain) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Requesting wildcard certificate for {domains}") - - if ( - certbot_new_with_retry( - "dns", - domains, - email, - provider, - credentials_file, - "default", - profile, - staging, - domains_to_ask.get(base_domain, 0) == 2, - cmd_env=env, - ca_provider=ca_provider, - api_key=api_key, - server_name=original_server, - ) - != 0 - ): + LOGGER.debug(f"Requesting wildcard certificate for " + f"{domains}") + + wildcard_result = certbot_new_with_retry( + "dns", + domains, + email, + provider, + credentials_file, + "default", + profile, + staging, + domains_to_ask.get(base_domain, 0) == 2, + cmd_env=env, + ca_provider=ca_provider, + api_key=api_key, + server_name=original_server, + ) + + if wildcard_result != 0: status = 2 certificates_failed += 1 - LOGGER.error(f"Certificate generation failed for domain(s) {domains}...") + LOGGER.error(f"Certificate generation failed for " + f"domain(s) {domains}...") else: status = 1 if status == 0 else status certificates_generated += 1 - LOGGER.info(f"Certificate generation succeeded for domain(s): {domains}") + LOGGER.info(f"Certificate generation succeeded for " + f"domain(s): {domains}") generated_domains.update(domains_split) else: LOGGER.info( - "No wildcard domains found, skipping wildcard certificate(s) generation..." + "No wildcard domains found, skipping wildcard certificate(s) " + "generation..." ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": @@ -2470,10 +2699,11 @@ def certbot_new( # If the file is not in the wildcard groups, remove it if file not in credential_paths: LOGGER.info(f"Removing old credentials file {file}") + service_id = (file.parent.name + if file.parent.name != "letsencrypt" else "") JOB.del_cache( file.name, job_name="certbot-renew", - service_id=(file.parent.name - if file.parent.name != "letsencrypt" else "") + service_id=service_id ) cleaned_files += 1 @@ -2496,7 +2726,7 @@ def certbot_new( CERTBOT_BIN, "certificates", "--config-dir", - DATA_PATH.as_posix(), + str(DATA_PATH), "--work-dir", WORK_DIR, "--logs-dir", @@ -2515,8 +2745,10 @@ def certbot_new( certificates_removed = 0 if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found {len(certificate_blocks)} certificates to evaluate") - LOGGER.debug(f"Active certificates: {sorted(active_cert_names)}") + LOGGER.debug(f"Found {len(certificate_blocks)} certificates " + f"to evaluate") + LOGGER.debug(f"Active certificates: " + f"{sorted(str(name) for name in active_cert_names)}") for block in certificate_blocks: cert_name = block.split("\n", 1)[0].strip() @@ -2537,7 +2769,7 @@ def certbot_new( CERTBOT_BIN, "delete", "--config-dir", - DATA_PATH.as_posix(), + str(DATA_PATH), "--work-dir", WORK_DIR, "--logs-dir", @@ -2555,11 +2787,13 @@ def certbot_new( ) if delete_proc.returncode == 0: - LOGGER.info(f"Successfully deleted certificate {cert_name}") + LOGGER.info(f"Successfully deleted certificate " + f"{cert_name}") certificates_removed += 1 cert_dir = DATA_PATH.joinpath("live", cert_name) archive_dir = DATA_PATH.joinpath("archive", cert_name) - renewal_file = DATA_PATH.joinpath("renewal", f"{cert_name}.conf") + renewal_file = DATA_PATH.joinpath("renewal", + f"{cert_name}.conf") for path in (cert_dir, archive_dir): if path.exists(): try: @@ -2567,15 +2801,18 @@ def certbot_new( try: file.unlink() except Exception as e: - LOGGER.error(f"Failed to remove file {file}: {e}") + LOGGER.error(f"Failed to remove file " + f"{file}: {e}") path.rmdir() LOGGER.info(f"Removed directory {path}") except Exception as e: - LOGGER.error(f"Failed to remove directory {path}: {e}") + LOGGER.error(f"Failed to remove directory " + f"{path}: {e}") if renewal_file.exists(): try: renewal_file.unlink() - LOGGER.info(f"Removed renewal file {renewal_file}") + LOGGER.info(f"Removed renewal file " + f"{renewal_file}") except Exception as e: LOGGER.error( f"Failed to remove renewal file " @@ -2588,7 +2825,8 @@ def certbot_new( ) if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate cleanup completed - removed {certificates_removed} certificates") + LOGGER.debug(f"Certificate cleanup completed - removed " + f"{certificates_removed} certificates") else: LOGGER.error(f"Error listing certificates: {proc.stdout}") @@ -2620,6 +2858,7 @@ def certbot_new( LOGGER.debug("Script failed with unexpected exception") if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate generation process completed with status: {status}") + LOGGER.debug(f"Certificate generation process completed with status: " + f"{status}") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index c0cc8171ce..ab92b633e0 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -123,8 +123,9 @@ ) # Pass database URI if configured (for cluster deployments) - if getenv("DATABASE_URI"): - env["DATABASE_URI"] = getenv("DATABASE_URI") + database_uri = getenv("DATABASE_URI") + if database_uri is not None: + env["DATABASE_URI"] = database_uri if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": LOGGER.debug("Environment configuration for certbot:") @@ -137,7 +138,7 @@ LOGGER.debug(f" PYTHONPATH: {pythonpath_display}") LOGGER.debug(f" RELOAD_MIN_TIMEOUT: {env['RELOAD_MIN_TIMEOUT']}") LOGGER.debug(f" DISABLE_CONFIGURATION_TESTING: {env['DISABLE_CONFIGURATION_TESTING']}") - LOGGER.debug(f" DATABASE_URI configured: {'Yes' if getenv('DATABASE_URI') else 'No'}") + LOGGER.debug(f" DATABASE_URI configured: {'Yes' if database_uri else 'No'}") # Construct certbot renew command with appropriate options # --no-random-sleep-on-renew: Prevents random delays in scheduled runs diff --git a/src/common/core/letsencrypt/ui/actions.py b/src/common/core/letsencrypt/ui/actions.py index 478a98d17d..7f261833cf 100644 --- a/src/common/core/letsencrypt/ui/actions.py +++ b/src/common/core/letsencrypt/ui/actions.py @@ -34,7 +34,8 @@ def extract_cache(folder_path, cache_files): if is_debug: logger.debug(f"Created directory structure: {folder_path}") - logger.debug(f"Directory permissions: {oct(folder_path.stat().st_mode)}") + logger.debug(f"Directory permissions: " + f"{oct(folder_path.stat().st_mode)}") extracted_files = 0 total_bytes = 0 @@ -44,7 +45,8 @@ def extract_cache(folder_path, cache_files): file_data = cache_file.get("data", b"") if is_debug: - logger.debug(f"Examining cache file {i+1}/{len(cache_files)}: {file_name}") + logger.debug(f"Examining cache file {i+1}/{len(cache_files)}: " + f"{file_name}") logger.debug(f"File size: {len(file_data)} bytes") if (cache_file["file_name"].endswith(".tgz") and @@ -60,36 +62,46 @@ def extract_cache(folder_path, cache_files): members = tar.getmembers() if is_debug: - logger.debug(f"Archive contains {len(members)} members") + logger.debug(f"Archive contains {len(members)} " + f"members") # Show first few members for j, member in enumerate(members[:5]): - logger.debug(f" Member {j+1}: {member.name} " - f"({member.size} bytes, " - f"{'dir' if member.isdir() else 'file'})") + logger.debug( + f" Member {j+1}: {member.name} " + f"({member.size} bytes, " + f"{'dir' if member.isdir() else 'file'})" + ) if len(members) > 5: - logger.debug(f" ... and {len(members) - 5} more members") + logger.debug(f" ... and {len(members) - 5} " + f"more members") try: tar.extractall(folder_path, filter="fully_trusted") if is_debug: - logger.debug("Extraction completed with fully_trusted filter") + logger.debug("Extraction completed with " + "fully_trusted filter") except TypeError: # Fallback for older Python versions without filter if is_debug: - logger.debug("Using fallback extraction without filter") + logger.debug("Using fallback extraction without " + "filter") tar.extractall(folder_path) extracted_files += 1 total_bytes += len(cache_file['data']) if is_debug: - logger.debug(f"Successfully extracted {cache_file['file_name']}") - logger.debug(f"Extracted {len(members)} items from archive") + logger.debug(f"Successfully extracted " + f"{cache_file['file_name']}") + logger.debug(f"Extracted {len(members)} items from " + f"archive") except Exception as e: - logger.error(f"Failed to extract {cache_file['file_name']}: {e}") + logger.error(f"Failed to extract {cache_file['file_name']}: " + f"{e}") if is_debug: - logger.debug(f"Extraction error details: {format_exc()}") + logger.debug(f"Extraction error details: " + f"{format_exc()}") else: if is_debug: logger.debug(f"Skipping non-archive file: {file_name}") @@ -114,7 +126,8 @@ def extract_cache(folder_path, cache_files): # Show some example files for i, file_item in enumerate(files[:5]): rel_path = file_item.relative_to(folder_path) - logger.debug(f" File {i+1}: {rel_path} ({file_item.stat().st_size} bytes)") + logger.debug(f" File {i+1}: {rel_path} " + f"({file_item.stat().st_size} bytes)") if len(files) > 5: logger.debug(f" ... and {len(files) - 5} more files") @@ -138,8 +151,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: is_debug = getenv("LOG_LEVEL") == "debug" if is_debug: - logger.debug(f"Retrieving certificate info from {len(folder_paths)} " - f"folder paths") + logger.debug(f"Retrieving certificate info from " + f"{len(folder_paths)} folder paths") certificates = { "domain": [], @@ -161,8 +174,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: for folder_idx, folder_path in enumerate(folder_paths): if is_debug: - logger.debug(f"Processing folder {folder_idx + 1}/{len(folder_paths)}: " - f"{folder_path}") + logger.debug(f"Processing folder {folder_idx + 1}/" + f"{len(folder_paths)}: {folder_path}") cert_files = list(folder_path.joinpath("live").glob("*/fullchain.pem")) @@ -176,8 +189,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: total_certs_processed += 1 if is_debug: - logger.debug(f"Processing certificate {total_certs_processed}: " - f"{domain}") + logger.debug(f"Processing certificate " + f"{total_certs_processed}: {domain}") # Initialize default certificate information cert_info = { @@ -199,20 +212,25 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Parse the certificate file try: if is_debug: - logger.debug(f"Loading X.509 certificate from {cert_file}") - logger.debug(f"Certificate file size: {cert_file.stat().st_size} bytes") + logger.debug(f"Loading X.509 certificate from " + f"{cert_file}") + logger.debug(f"Certificate file size: " + f"{cert_file.stat().st_size} bytes") cert_bytes = cert_file.read_bytes() if is_debug: - logger.debug(f"Read {len(cert_bytes)} bytes from certificate file") - logger.debug(f"Certificate data preview: {cert_bytes[:100]}...") + logger.debug(f"Read {len(cert_bytes)} bytes from " + f"certificate file") + logger.debug(f"Certificate data preview: " + f"{cert_bytes[:100]}...") cert = x509.load_pem_x509_certificate( cert_bytes, default_backend() ) if is_debug: - logger.debug(f"Successfully loaded certificate for {domain}") + logger.debug(f"Successfully loaded certificate for " + f"{domain}") logger.debug(f"Certificate version: {cert.version}") logger.debug(f"Certificate serial: {cert.serial_number}") @@ -223,10 +241,12 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: if subject: cert_info["common_name"] = subject[0].value if is_debug: - logger.debug(f"Certificate CN: {cert_info['common_name']}") + logger.debug(f"Certificate CN: " + f"{cert_info['common_name']}") else: if is_debug: - logger.debug("No Common Name found in certificate subject") + logger.debug("No Common Name found in certificate " + "subject") logger.debug(f"Full subject: {cert.subject}") # Extract issuer (Certificate Authority) @@ -236,10 +256,12 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: if issuer: cert_info["issuer"] = issuer[0].value if is_debug: - logger.debug(f"Certificate issuer: {cert_info['issuer']}") + logger.debug(f"Certificate issuer: " + f"{cert_info['issuer']}") else: if is_debug: - logger.debug("No Common Name found in certificate issuer") + logger.debug("No Common Name found in certificate " + "issuer") logger.debug(f"Full issuer: {cert.issuer}") # Extract validity period @@ -251,12 +273,14 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: ) if is_debug: - logger.debug(f"Certificate validity: {cert_info['valid_from']} " - f"to {cert_info['valid_to']}") + logger.debug(f"Certificate validity: " + f"{cert_info['valid_from']} to " + f"{cert_info['valid_to']}") # Check if certificate is currently valid from datetime import datetime, timezone now = datetime.now(timezone.utc) - is_valid = cert.not_valid_before <= now <= cert.not_valid_after + is_valid = (cert.not_valid_before <= now <= + cert.not_valid_after) logger.debug(f"Certificate currently valid: {is_valid}") # Extract serial number @@ -267,12 +291,13 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: cert_info["fingerprint"] = fingerprint_bytes.hex() if is_debug: - logger.debug(f"Certificate fingerprint: {cert_info['fingerprint']}") + logger.debug(f"Certificate fingerprint: " + f"{cert_info['fingerprint']}") # Extract version cert_info["version"] = cert.version.name - # Check for OCSP support via Authority Information Access extension + # Check for OCSP support via Authority Information Access ext try: aia_ext = cert.extensions.get_extension_for_oid( oid.ExtensionOID.AUTHORITY_INFORMATION_ACCESS @@ -281,36 +306,45 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: for access_description in aia_ext.value: if (access_description.access_method == oid.AuthorityInformationAccessOID.OCSP): - ocsp_urls.append(str(access_description.access_location.value)) + ocsp_urls.append( + str(access_description.access_location.value) + ) if ocsp_urls: cert_info["ocsp_support"] = "Yes" if is_debug: - logger.debug(f"OCSP URLs found for {domain}: {ocsp_urls}") + logger.debug(f"OCSP URLs found for {domain}: " + f"{ocsp_urls}") else: cert_info["ocsp_support"] = "No" if is_debug: - logger.debug(f"AIA extension found for {domain} but no OCSP URLs") + logger.debug(f"AIA extension found for {domain} " + f"but no OCSP URLs") except x509.ExtensionNotFound: cert_info["ocsp_support"] = "No" if is_debug: - logger.debug(f"No Authority Information Access extension found for {domain}") + logger.debug(f"No Authority Information Access " + f"extension found for {domain}") except Exception as ocsp_error: cert_info["ocsp_support"] = "Unknown" if is_debug: - logger.debug(f"Error checking OCSP support for {domain}: {ocsp_error}") + logger.debug(f"Error checking OCSP support for " + f"{domain}: {ocsp_error}") if is_debug: - logger.debug(f"Certificate processing completed for {domain}") + logger.debug(f"Certificate processing completed for " + f"{domain}") logger.debug(f" - Serial: {cert_info['serial_number']}") logger.debug(f" - Version: {cert_info['version']}") logger.debug(f" - Subject: {cert_info['common_name']}") logger.debug(f" - Issuer: {cert_info['issuer']}") - logger.debug(f" - OCSP Support: {cert_info['ocsp_support']}") + logger.debug(f" - OCSP Support: " + f"{cert_info['ocsp_support']}") except BaseException as e: - error_msg = f"Error while parsing certificate {cert_file}: {e}" + error_msg = (f"Error while parsing certificate {cert_file}: " + f"{e}") logger.error(error_msg) if is_debug: logger.debug(f"Certificate parsing error details:") @@ -323,7 +357,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Parse the renewal configuration file try: - renewal_file = folder_path.joinpath("renewal", f"{domain}.conf") + renewal_file = folder_path.joinpath("renewal", + f"{domain}.conf") if renewal_file.exists(): if is_debug: @@ -372,7 +407,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: f"{renewal_file}: {e}") logger.error(error_msg) if is_debug: - logger.debug(f"Renewal config parsing error: {format_exc()}") + logger.debug(f"Renewal config parsing error: " + f"{format_exc()}") # Append all certificate information to the results for key in cert_info: @@ -386,7 +422,9 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Summary of OCSP support ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} for ocsp_status in certificates.get('ocsp_support', []): - ocsp_support_counts[ocsp_status] = ocsp_support_counts.get(ocsp_status, 0) + 1 + ocsp_support_counts[ocsp_status] = ( + ocsp_support_counts.get(ocsp_status, 0) + 1 + ) logger.debug(f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -449,12 +487,14 @@ def pre_render(app, *args, **kwargs): try: if is_debug: logger.debug("Starting Let's Encrypt data retrieval process") - logger.debug(f"Database connection available: {'db' in kwargs}") + logger.debug(f"Database connection available: " + f"{'db' in kwargs}") logger.debug(f"Root folder: {root_folder}") # Retrieve cache files from database if is_debug: - logger.debug("Fetching cache files from database for job: certbot-renew") + logger.debug("Fetching cache files from database for job: " + "certbot-renew") regular_cache_files = kwargs["db"].get_jobs_cache_files( job_name="certbot-renew" @@ -465,7 +505,8 @@ def pre_render(app, *args, **kwargs): for i, cache_file in enumerate(regular_cache_files): file_name = cache_file.get("file_name", "unknown") file_size = len(cache_file.get("data", b"")) - logger.debug(f" Cache file {i+1}: {file_name} ({file_size} bytes)") + logger.debug(f" Cache file {i+1}: {file_name} " + f"({file_size} bytes)") # Create unique temporary folder for extraction folder_uuid = str(uuid4()) @@ -484,7 +525,8 @@ def pre_render(app, *args, **kwargs): extract_cache(regular_le_folder, regular_cache_files) if is_debug: - logger.debug("Cache extraction completed, starting certificate parsing") + logger.debug("Cache extraction completed, starting certificate " + "parsing") # Parse certificates and retrieve information cert_data = retrieve_certificates_info((regular_le_folder,)) @@ -502,7 +544,8 @@ def pre_render(app, *args, **kwargs): for key in cert_data: value = cert_data[key][0] if cert_data[key] else "None" if key == "ocsp_support": - logger.debug(f" {key}: {value} (OCSP support detected)") + logger.debug(f" {key}: {value} (OCSP support " + f"detected)") else: logger.debug(f" {key}: {value}") @@ -513,7 +556,8 @@ def pre_render(app, *args, **kwargs): if is_debug: logger.debug(f"Return data structure keys: {list(ret.keys())}") - logger.debug(f"Certificate list structure: {list(ret['list_certificates'].keys())}") + logger.debug(f"Certificate list structure: " + f"{list(ret['list_certificates'].keys())}") except BaseException as e: error_msg = f"Failed to get Let's Encrypt certificates: {e}" @@ -524,18 +568,21 @@ def pre_render(app, *args, **kwargs): logger.debug(f" - Error type: {type(e).__name__}") logger.debug(f" - Error message: {str(e)}") logger.debug(f" - Error traceback: {format_exc()}") - logger.debug(f" - kwargs keys: {list(kwargs.keys()) if kwargs else 'None'}") + logger.debug(f" - kwargs keys: " + f"{list(kwargs.keys()) if kwargs else 'None'}") if "db" in kwargs: - logger.debug(f" - Database object type: {type(kwargs['db'])}") + logger.debug(f" - Database object type: " + f"{type(kwargs['db'])}") - ret["error"] = str(e) + ret["error"] = {"message": str(e)} finally: # Clean up temporary files if folder_path and folder_path.exists(): try: if is_debug: - logger.debug(f"Cleaning up temporary folder: {root_folder}") + logger.debug(f"Cleaning up temporary folder: " + f"{root_folder}") rmtree(root_folder, ignore_errors=True) diff --git a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py index a22ba86579..0d8c8c318f 100644 --- a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py +++ b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py @@ -76,7 +76,8 @@ def download_certificates(): cache_file["file_name"].startswith("folder:")): if is_debug: - LOGGER.debug(f"Extracting cache file: {cache_file['file_name']}") + LOGGER.debug(f"Extracting cache file: " + f"{cache_file['file_name']}") LOGGER.debug(f"File size: {len(cache_file['data'])} bytes") try: @@ -84,35 +85,43 @@ def download_certificates(): mode="r:gz") as tar: member_count = len(tar.getmembers()) if is_debug: - LOGGER.debug(f"Archive contains {member_count} members") + LOGGER.debug(f"Archive contains {member_count} " + f"members") try: tar.extractall(DATA_PATH, filter="fully_trusted") if is_debug: - LOGGER.debug("Extraction completed with fully_trusted filter") + LOGGER.debug("Extraction completed with " + "fully_trusted filter") except TypeError: if is_debug: - LOGGER.debug("Falling back to extraction without filter") + LOGGER.debug("Falling back to extraction without " + "filter") tar.extractall(DATA_PATH) extracted_count += 1 if is_debug: - LOGGER.debug(f"Successfully extracted {cache_file['file_name']}") + LOGGER.debug(f"Successfully extracted " + f"{cache_file['file_name']}") except Exception as e: - LOGGER.error(f"Failed to extract {cache_file['file_name']}: {e}") + LOGGER.error(f"Failed to extract {cache_file['file_name']}: " + f"{e}") if is_debug: LOGGER.debug(f"Extraction error details: {format_exc()}") else: if is_debug: - LOGGER.debug(f"Skipping non-matching file: {cache_file['file_name']}") + LOGGER.debug(f"Skipping non-matching file: " + f"{cache_file['file_name']}") if is_debug: - LOGGER.debug(f"Certificate download completed: {extracted_count} files extracted") + LOGGER.debug(f"Certificate download completed: {extracted_count} " + f"files extracted") # List extracted directory contents if Path(DATA_PATH).exists(): contents = list(Path(DATA_PATH).rglob("*")) - LOGGER.debug(f"Extracted directory contains {len(contents)} items") + LOGGER.debug(f"Extracted directory contains {len(contents)} " + f"items") for item in contents[:10]: # Show first 10 items LOGGER.debug(f" - {item}") if len(contents) > 10: @@ -163,9 +172,11 @@ def retrieve_certificates(): certificates["domain"].append(domain) if is_debug: - LOGGER.debug(f"Processing certificate {len(certificates['domain'])}: {domain}") + LOGGER.debug(f"Processing certificate " + f"{len(certificates['domain'])}: {domain}") LOGGER.debug(f"Certificate file path: {cert_file}") - LOGGER.debug(f"Certificate file size: {cert_file.stat().st_size} bytes") + LOGGER.debug(f"Certificate file size: " + f"{cert_file.stat().st_size} bytes") cert_info = { "common_name": "Unknown", @@ -189,7 +200,8 @@ def retrieve_certificates(): cert_data = cert_file.read_bytes() if is_debug: - LOGGER.debug(f"Certificate data length: {len(cert_data)} bytes") + LOGGER.debug(f"Certificate data length: {len(cert_data)} " + f"bytes") LOGGER.debug(f"Certificate starts with: {cert_data[:50]}") cert = x509.load_pem_x509_certificate( @@ -207,10 +219,12 @@ def retrieve_certificates(): if subject: cert_info["common_name"] = subject[0].value if is_debug: - LOGGER.debug(f"Certificate CN extracted: {cert_info['common_name']}") + LOGGER.debug(f"Certificate CN extracted: " + f"{cert_info['common_name']}") else: if is_debug: - LOGGER.debug("No Common Name found in certificate subject") + LOGGER.debug("No Common Name found in certificate " + "subject") issuer = cert.issuer.get_attributes_for_oid( x509.NameOID.COMMON_NAME @@ -218,10 +232,12 @@ def retrieve_certificates(): if issuer: cert_info["issuer"] = issuer[0].value if is_debug: - LOGGER.debug(f"Certificate issuer extracted: {cert_info['issuer']}") + LOGGER.debug(f"Certificate issuer extracted: " + f"{cert_info['issuer']}") else: if is_debug: - LOGGER.debug("No Common Name found in certificate issuer") + LOGGER.debug("No Common Name found in certificate " + "issuer") cert_info["valid_from"] = ( cert.not_valid_before.astimezone().isoformat() @@ -232,7 +248,8 @@ def retrieve_certificates(): if is_debug: LOGGER.debug(f"Certificate validity period: " - f"{cert_info['valid_from']} to {cert_info['valid_to']}") + f"{cert_info['valid_from']} to " + f"{cert_info['valid_to']}") cert_info["serial_number"] = str(cert.serial_number) cert_info["fingerprint"] = cert.fingerprint(hashes.SHA256()).hex() @@ -247,7 +264,9 @@ def retrieve_certificates(): for access_description in aia_ext.value: if (access_description.access_method == oid.AuthorityInformationAccessOID.OCSP): - ocsp_urls.append(str(access_description.access_location.value)) + ocsp_urls.append( + str(access_description.access_location.value) + ) if ocsp_urls: cert_info["ocsp_support"] = "Yes" @@ -261,16 +280,19 @@ def retrieve_certificates(): except x509.ExtensionNotFound: cert_info["ocsp_support"] = "No" if is_debug: - LOGGER.debug("No Authority Information Access extension found") + LOGGER.debug("No Authority Information Access extension " + "found") except Exception as ocsp_error: cert_info["ocsp_support"] = "Unknown" if is_debug: - LOGGER.debug(f"Error checking OCSP support: {ocsp_error}") + LOGGER.debug(f"Error checking OCSP support: " + f"{ocsp_error}") if is_debug: LOGGER.debug(f"Certificate details extracted:") LOGGER.debug(f" - Serial: {cert_info['serial_number']}") - LOGGER.debug(f" - Fingerprint: {cert_info['fingerprint'][:16]}...") + LOGGER.debug(f" - Fingerprint: " + f"{cert_info['fingerprint'][:16]}...") LOGGER.debug(f" - Version: {cert_info['version']}") LOGGER.debug(f" - OCSP Support: {cert_info['ocsp_support']}") @@ -278,15 +300,18 @@ def retrieve_certificates(): LOGGER.debug(format_exc()) LOGGER.error(f"Error while parsing certificate {cert_file}: {e}") if is_debug: - LOGGER.debug(f"Certificate parsing failed for {domain}: {str(e)}") + LOGGER.debug(f"Certificate parsing failed for {domain}: " + f"{str(e)}") LOGGER.debug(f"Error type: {type(e).__name__}") try: - renewal_file = Path(DATA_PATH).joinpath("renewal", f"{domain}.conf") + renewal_file = Path(DATA_PATH).joinpath("renewal", + f"{domain}.conf") if renewal_file.exists(): if is_debug: LOGGER.debug(f"Processing renewal file: {renewal_file}") - LOGGER.debug(f"Renewal file size: {renewal_file.stat().st_size} bytes") + LOGGER.debug(f"Renewal file size: " + f"{renewal_file.stat().st_size} bytes") config_lines_processed = 0 with renewal_file.open("r") as f: @@ -296,57 +321,69 @@ def retrieve_certificates(): continue config_lines_processed += 1 - if is_debug and line_num <= 10: # Debug first 10 meaningful lines - LOGGER.debug(f"Renewal config line {line_num}: {line}") + if is_debug and line_num <= 10: # Debug first 10 lines + LOGGER.debug(f"Renewal config line {line_num}: " + f"{line}") if line.startswith("preferred_profile = "): cert_info["preferred_profile"] = ( line.split(" = ")[1].strip() ) if is_debug: - LOGGER.debug(f"Found preferred_profile: {cert_info['preferred_profile']}") + LOGGER.debug(f"Found preferred_profile: " + f"{cert_info['preferred_profile']}") elif line.startswith("pref_challs = "): challenges = line.split(" = ")[1].strip() cert_info["challenge"] = challenges.split(",")[0] if is_debug: - LOGGER.debug(f"Found challenge: {cert_info['challenge']} (from {challenges})") + LOGGER.debug(f"Found challenge: " + f"{cert_info['challenge']} " + f"(from {challenges})") elif line.startswith("authenticator = "): cert_info["authenticator"] = ( line.split(" = ")[1].strip() ) if is_debug: - LOGGER.debug(f"Found authenticator: {cert_info['authenticator']}") + LOGGER.debug(f"Found authenticator: " + f"{cert_info['authenticator']}") elif line.startswith("server = "): cert_info["issuer_server"] = ( line.split(" = ")[1].strip() ) if is_debug: - LOGGER.debug(f"Found issuer_server: {cert_info['issuer_server']}") + LOGGER.debug(f"Found issuer_server: " + f"{cert_info['issuer_server']}") elif line.startswith("key_type = "): cert_info["key_type"] = ( line.split(" = ")[1].strip() ) if is_debug: - LOGGER.debug(f"Found key_type: {cert_info['key_type']}") + LOGGER.debug(f"Found key_type: " + f"{cert_info['key_type']}") if is_debug: - LOGGER.debug(f"Processed {config_lines_processed} configuration lines") + LOGGER.debug(f"Processed {config_lines_processed} " + f"configuration lines") LOGGER.debug(f"Final renewal configuration for {domain}:") - LOGGER.debug(f" - Profile: {cert_info['preferred_profile']}") + LOGGER.debug(f" - Profile: " + f"{cert_info['preferred_profile']}") LOGGER.debug(f" - Challenge: {cert_info['challenge']}") - LOGGER.debug(f" - Authenticator: {cert_info['authenticator']}") + LOGGER.debug(f" - Authenticator: " + f"{cert_info['authenticator']}") LOGGER.debug(f" - Server: {cert_info['issuer_server']}") LOGGER.debug(f" - Key type: {cert_info['key_type']}") else: if is_debug: - LOGGER.debug(f"No renewal file found for {domain} at {renewal_file}") + LOGGER.debug(f"No renewal file found for {domain} at " + f"{renewal_file}") except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while parsing renewal configuration " f"{renewal_file}: {e}") if is_debug: - LOGGER.debug(f"Renewal config parsing failed for {domain}: {str(e)}") + LOGGER.debug(f"Renewal config parsing failed for {domain}: " + f"{str(e)}") LOGGER.debug(f"Error type: {type(e).__name__}") for key in cert_info: @@ -357,7 +394,9 @@ def retrieve_certificates(): # Summary of OCSP support ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} for ocsp_status in certificates.get('ocsp_support', []): - ocsp_support_counts[ocsp_status] = ocsp_support_counts.get(ocsp_status, 0) + 1 + ocsp_support_counts[ocsp_status] = ( + ocsp_support_counts.get(ocsp_status, 0) + 1 + ) LOGGER.debug(f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -401,7 +440,8 @@ def letsencrypt_fetch(): certs = retrieve_certificates() if is_debug: - LOGGER.debug(f"Retrieved certificates: {len(certs.get('domain', []))}") + LOGGER.debug(f"Retrieved certificates: " + f"{len(certs.get('domain', []))}") for i, domain in enumerate(certs.get("domain", [])): cert_data = { @@ -424,7 +464,8 @@ def letsencrypt_fetch(): if is_debug: LOGGER.debug(f"Added certificate to list: {domain}") - LOGGER.debug(f" - OCSP Support: {cert_data['ocsp_support']}") + LOGGER.debug(f" - OCSP Support: " + f"{cert_data['ocsp_support']}") LOGGER.debug(f" - Challenge: {cert_data['challenge']}") LOGGER.debug(f" - Key Type: {cert_data['key_type']}") @@ -440,7 +481,8 @@ def letsencrypt_fetch(): } if is_debug: - LOGGER.debug(f"Returning {len(cert_list)} certificates to DataTables") + LOGGER.debug(f"Returning {len(cert_list)} certificates to " + f"DataTables") return jsonify(response_data) @@ -514,7 +556,8 @@ def letsencrypt_delete(): # Clean up certificate directories and files cert_dir = Path(DATA_PATH).joinpath("live", cert_name) archive_dir = Path(DATA_PATH).joinpath("archive", cert_name) - renewal_file = Path(DATA_PATH).joinpath("renewal", f"{cert_name}.conf") + renewal_file = Path(DATA_PATH).joinpath("renewal", + f"{cert_name}.conf") if is_debug: LOGGER.debug(f"Cleaning up directories for {cert_name}") @@ -528,7 +571,8 @@ def letsencrypt_delete(): if is_debug: LOGGER.debug(f"Removed file: {file}") except Exception as e: - LOGGER.error(f"Failed to remove file {file}: {e}") + LOGGER.error(f"Failed to remove file {file}: " + f"{e}") path.rmdir() LOGGER.info(f"Removed directory {path}") except Exception as e: @@ -585,7 +629,8 @@ def letsencrypt_delete(): "message": f"Successfully deleted certificate " f"{cert_name}"}) else: - error_msg = f"Failed to delete certificate {cert_name}: {delete_proc.stdout}" + error_msg = (f"Failed to delete certificate {cert_name}: " + f"{delete_proc.stdout}") LOGGER.error(error_msg) return jsonify({"status": "ko", "message": error_msg}) From ccb2fd123cd303a1217fa48a5daaf6a0bfb4728b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 11:33:59 +0200 Subject: [PATCH 09/15] fix mixed up comments seperator --- .../ui/blueprints/static/js/main.js | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js index 6e3112e7a6..f978202eab 100644 --- a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js +++ b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js @@ -1,15 +1,15 @@ (async function waitForDependencies() { - # Wait for jQuery + // Wait for jQuery while (typeof jQuery === "undefined") { await new Promise((resolve) => setTimeout(resolve, 100)); } - # Wait for $ to be available (in case of jQuery.noConflict()) + // Wait for $ to be available (in case of jQuery.noConflict()) while (typeof $ === "undefined") { await new Promise((resolve) => setTimeout(resolve, 100)); } - # Wait for DataTable to be available + // Wait for DataTable to be available while (typeof $.fn.DataTable === "undefined") { await new Promise((resolve) => setTimeout(resolve, 100)); } @@ -25,7 +25,7 @@ console.debug("DataTables version:", $.fn.DataTable.version); } - # Ensure i18next is loaded before using it + // Ensure i18next is loaded before using it const t = typeof i18next !== "undefined" ? i18next.t @@ -82,7 +82,7 @@ }, ]; - # Set up the delete confirmation modal for certificates + // Set up the delete confirmation modal for certificates const setupDeleteCertModal = (certs) => { if (isDebug) { console.debug("Setting up delete modal for certificates:", @@ -128,7 +128,7 @@ } }; - # Show error modal with title and message + // Show error modal with title and message const showErrorModal = (title, message) => { if (isDebug) { console.debug("Showing error modal:", title, message); @@ -142,7 +142,7 @@ errorModal.show(); }; - # Handle delete button click events + // Handle delete button click events $("#confirmDeleteCertBtn").on("click", function () { const certName = $(this).data("cert-name"); const certNames = $(this).data("cert-names"); @@ -155,7 +155,7 @@ if (certName) { deleteCertificate(certName); } else if (certNames && Array.isArray(certNames)) { - # Delete multiple certificates sequentially + // Delete multiple certificates sequentially const deleteNext = (index) => { if (index < certNames.length) { deleteCertificate(certNames[index], () => { @@ -172,7 +172,7 @@ $("#deleteCertModal").modal("hide"); }); - # Delete a single certificate with optional callback + // Delete a single certificate with optional callback function deleteCertificate(certName, callback) { if (isDebug) { console.debug("Starting certificate deletion process:"); @@ -299,14 +299,14 @@ }); } - # DataTable Layout and Button configuration + // DataTable Layout and Button configuration const layout = { top1: { searchPanes: { viewTotal: true, cascadePanes: true, collapse: false, - # Issuer, Preferred Profile, Challenge, Key Type, and OCSP + // Issuer, Preferred Profile, Challenge, Key Type, and OCSP columns: [2, 5, 6, 7, 8], }, }, @@ -458,7 +458,7 @@ const sessionAutoRefresh = sessionStorage.getItem("letsencryptAutoRefresh"); - # Toggle auto-refresh functionality + // Toggle auto-refresh functionality function toggleAutoRefresh() { autoRefresh = !autoRefresh; sessionStorage.setItem("letsencryptAutoRefresh", autoRefresh); @@ -502,7 +502,7 @@ toggleAutoRefresh(); } - # Get currently selected certificates from DataTable + // Get currently selected certificates from DataTable const getSelectedCertificates = () => { const certs = []; $("tr.selected").each(function () { @@ -518,7 +518,7 @@ return certs; }; - # Custom DataTable button for auto-refresh + // Custom DataTable button for auto-refresh $.fn.dataTable.ext.buttons.auto_refresh = { text: ( '' + @@ -530,7 +530,7 @@ }, }; - # Custom DataTable button for certificate deletion + // Custom DataTable button for certificate deletion $.fn.dataTable.ext.buttons.delete_cert = { text: ( `` + @@ -568,7 +568,7 @@ }, }; - # Build column definitions for DataTable + // Build column definitions for DataTable function buildColumnDefs() { return [ { @@ -639,7 +639,7 @@ ]; } - # Define the columns for the DataTable + // Define the columns for the DataTable function buildColumns() { return [ { @@ -664,7 +664,7 @@ ]; } - # Manage header tooltips for DataTable columns + // Manage header tooltips for DataTable columns function updateHeaderTooltips(selector, headers) { $(selector) .find("th") @@ -684,7 +684,7 @@ $('[data-bs-toggle="tooltip"]').tooltip("dispose").tooltip(); } - # Initialize the DataTable with complete configuration + // Initialize the DataTable with complete configuration const letsencrypt_config = { tableSelector: "#letsencrypt", tableName: "letsencrypt", @@ -791,7 +791,7 @@ $(".tooltip").remove(); }); - # Toggle action button based on selection + // Toggle action button based on selection dt.on("select.dt deselect.dt", function () { const count = dt.rows({ selected: true }).count(); $(".action-button").toggleClass("disabled", count === 0); From 9ac3a2a81a41bc8bd52d2456400165f1a3d225e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Sun, 29 Jun 2025 11:47:13 +0200 Subject: [PATCH 10/15] fix markdownlint error messages --- src/common/core/letsencrypt/README.md | 41 ++++++++++++++------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/common/core/letsencrypt/README.md b/src/common/core/letsencrypt/README.md index f81dab023e..9258e98436 100644 --- a/src/common/core/letsencrypt/README.md +++ b/src/common/core/letsencrypt/README.md @@ -1,3 +1,5 @@ +# Let's Encrypt Plugin + The Let's Encrypt plugin simplifies SSL/TLS certificate management by automating the creation, renewal, and configuration of free certificates from multiple certificate authorities. This feature enables secure HTTPS connections for your websites without the complexity of manual certificate management, reducing both cost and administrative overhead. **How it works:** @@ -12,7 +14,7 @@ The Let's Encrypt plugin simplifies SSL/TLS certificate management by automating !!! info "Prerequisites" To use this feature, ensure that proper DNS **A records** are configured for each domain, pointing to the public IP(s) where BunkerWeb is accessible. Without correct DNS configuration, the domain verification process will fail. -### How to Use +## How to Use Follow these steps to configure and use the Let's Encrypt feature: @@ -29,7 +31,7 @@ Follow these steps to configure and use the Let's Encrypt feature: The plugin supports multiple certificate authorities: - **Let's Encrypt**: Free, widely trusted, 90-day certificates - **ZeroSSL**: Free alternative with competitive rate limits, supports EAB (External Account Binding) - + ZeroSSL requires an API key for automated EAB credential generation. Without an API key, you can manually provide EAB credentials using `ACME_ZEROSSL_EAB_KID` and `ACME_ZEROSSL_EAB_HMAC_KEY`. !!! tip "Certificate Profiles" @@ -42,7 +44,7 @@ Follow these steps to configure and use the Let's Encrypt feature: !!! info "Profile Availability" Note that the `tlsserver` and `shortlived` profiles may not be available in all environments or with all ACME clients at this time. The `classic` profile has the widest compatibility and is recommended for most users. If a selected profile is not available, the system will automatically fall back to the `classic` profile. -### Advanced Security Features +## Advanced Security Features The plugin includes several advanced security and validation features: @@ -52,7 +54,7 @@ The plugin includes several advanced security and validation features: - **Retry Mechanisms**: Intelligent retry with exponential backoff for failed certificate generation attempts - **Certificate Key Types**: Supports both RSA and ECDSA keys with provider-specific optimizations -### Configuration Settings +## Configuration Settings | Setting | Default | Context | Multiple | Description | | ---------------------------------- | ------------------------ | --------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | @@ -108,32 +110,32 @@ The plugin includes several advanced security and validation features: !!! warning "Rate Limits" Certificate authorities impose rate limits on certificate issuance. When testing configurations, use the staging environment by setting `USE_LETS_ENCRYPT_STAGING` to `yes` to avoid hitting production rate limits. Staging certificates are not trusted by browsers but are useful for validating your setup. -### Supported DNS Providers +## Supported DNS Providers The Let's Encrypt plugin supports a wide range of DNS providers for DNS challenges. Each provider requires specific credentials that must be provided using the `LETS_ENCRYPT_DNS_CREDENTIAL_ITEM` setting. | Provider | Description | Mandatory Settings | Optional Settings | Documentation | | -------------- | --------------- | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------- | -| `cloudflare` | Cloudflare | either `api_token`
    or `email` and `api_key` | | [Documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/) | +| `cloudflare` | Cloudflare | either `api_token` or `email` and `api_key` | | [Documentation](https://certbot-dns-cloudflare.readthedocs.io/en/stable/) | | `desec` | deSEC | `token` | | [Documentation](https://github.com/desec-io/certbot-dns-desec/blob/main/README.md) | | `digitalocean` | DigitalOcean | `token` | | [Documentation](https://certbot-dns-digitalocean.readthedocs.io/en/stable/) | | `dnsimple` | DNSimple | `token` | | [Documentation](https://certbot-dns-dnsimple.readthedocs.io/en/stable/) | -| `dnsmadeeasy` | DNS Made Easy | `api_key`
    `secret_key` | | [Documentation](https://certbot-dns-dnsmadeeasy.readthedocs.io/en/stable/) | -| `gehirn` | Gehirn DNS | `api_token`
    `api_secret` | | [Documentation](https://certbot-dns-gehirn.readthedocs.io/en/stable/) | -| `google` | Google Cloud | `project_id`
    `private_key_id`
    `private_key`
    `client_email`
    `client_id`
    `client_x509_cert_url` | `type` (default: `service_account`)
    `auth_uri` (default: `https://accounts.google.com/o/oauth2/auth`)
    `token_uri` (default: `https://accounts.google.com/o/oauth2/token`)
    `auth_provider_x509_cert_url` (default: `https://www.googleapis.com/oauth2/v1/certs`) | [Documentation](https://certbot-dns-google.readthedocs.io/en/stable/) | +| `dnsmadeeasy` | DNS Made Easy | `api_key` `secret_key` | | [Documentation](https://certbot-dns-dnsmadeeasy.readthedocs.io/en/stable/) | +| `gehirn` | Gehirn DNS | `api_token` `api_secret` | | [Documentation](https://certbot-dns-gehirn.readthedocs.io/en/stable/) | +| `google` | Google Cloud | `project_id` `private_key_id` `private_key` `client_email` `client_id` `client_x509_cert_url` | `type` (default: `service_account`) `auth_uri` (default: `https://accounts.google.com/o/oauth2/auth`) `token_uri` (default: `https://accounts.google.com/o/oauth2/token`) `auth_provider_x509_cert_url` (default: `https://www.googleapis.com/oauth2/v1/certs`) | [Documentation](https://certbot-dns-google.readthedocs.io/en/stable/) | | `infomaniak` | Infomaniak | `token` | | [Documentation](https://github.com/infomaniak/certbot-dns-infomaniak) | -| `ionos` | IONOS | `prefix`
    `secret` | `endpoint` (default: `https://api.hosting.ionos.com`) | [Documentation](https://github.com/helgeerbe/certbot-dns-ionos/blob/master/README.md) | +| `ionos` | IONOS | `prefix` `secret` | `endpoint` (default: `https://api.hosting.ionos.com`) | [Documentation](https://github.com/helgeerbe/certbot-dns-ionos/blob/master/README.md) | | `linode` | Linode | `key` | | [Documentation](https://certbot-dns-linode.readthedocs.io/en/stable/) | -| `luadns` | LuaDNS | `email`
    `token` | | [Documentation](https://certbot-dns-luadns.readthedocs.io/en/stable/) | +| `luadns` | LuaDNS | `email` `token` | | [Documentation](https://certbot-dns-luadns.readthedocs.io/en/stable/) | | `njalla` | Njalla | `token` | | [Documentation](https://github.com/chaptergy/certbot-dns-njalla) | | `nsone` | NS1 | `api_key` | | [Documentation](https://certbot-dns-nsone.readthedocs.io/en/stable/) | -| `ovh` | OVH | `application_key`
    `application_secret`
    `consumer_key` | `endpoint` (default: `ovh-eu`) | [Documentation](https://certbot-dns-ovh.readthedocs.io/en/stable/) | -| `rfc2136` | RFC 2136 | `server`
    `name`
    `secret` | `port` (default: `53`)
    `algorithm` (default: `HMAC-SHA512`)
    `sign_query` (default: `false`) | [Documentation](https://certbot-dns-rfc2136.readthedocs.io/en/stable/) | -| `route53` | Amazon Route 53 | `access_key_id`
    `secret_access_key` | | [Documentation](https://certbot-dns-route53.readthedocs.io/en/stable/) | -| `sakuracloud` | Sakura Cloud | `api_token`
    `api_secret` | | [Documentation](https://certbot-dns-sakuracloud.readthedocs.io/en/stable/) | +| `ovh` | OVH | `application_key` `application_secret` `consumer_key` | `endpoint` (default: `ovh-eu`) | [Documentation](https://certbot-dns-ovh.readthedocs.io/en/stable/) | +| `rfc2136` | RFC 2136 | `server` `name` `secret` | `port` (default: `53`) `algorithm` (default: `HMAC-SHA512`) `sign_query` (default: `false`) | [Documentation](https://certbot-dns-rfc2136.readthedocs.io/en/stable/) | +| `route53` | Amazon Route 53 | `access_key_id` `secret_access_key` | | [Documentation](https://certbot-dns-route53.readthedocs.io/en/stable/) | +| `sakuracloud` | Sakura Cloud | `api_token` `api_secret` | | [Documentation](https://certbot-dns-sakuracloud.readthedocs.io/en/stable/) | | `scaleway` | Scaleway | `application_token` | | [Documentation](https://github.com/vanonox/certbot-dns-scaleway/blob/main/README.rst) | -### Certificate Key Types and Optimization +## Certificate Key Types and Optimization The plugin automatically selects optimal certificate key types based on the certificate authority and DNS provider: @@ -143,7 +145,7 @@ The plugin automatically selects optimal certificate key types based on the cert - **RSA Keys**: Used for specific providers that require them - Infomaniak and IONOS: RSA-4096 for compatibility -### Example Configurations +## Example Configurations === "Basic HTTP Challenge with Let's Encrypt" @@ -264,7 +266,7 @@ The plugin automatically selects optimal certificate key types based on the cert LETS_ENCRYPT_DNS_CREDENTIAL_ITEM_6: "client_x509_cert_url your-cert-url" ``` -### Troubleshooting +## Troubleshooting **Common Issues and Solutions:** @@ -294,9 +296,10 @@ The plugin automatically selects optimal certificate key types based on the cert **Debug Information:** Enable debug logging by setting `LOG_LEVEL: "DEBUG"` to get detailed information about: + - Certificate generation process - DNS validation steps - HTTP challenge deployment - CAA record checking - IP address validation -- Retry attempts and backoff timing \ No newline at end of file +- Retry attempts and backoff timing From e400de1134b938ed3c4269054ceefba6137c804e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Mon, 30 Jun 2025 12:50:43 +0200 Subject: [PATCH 11/15] consistent logging functions --- .../core/letsencrypt/jobs/certbot-auth.py | 168 ++-- .../core/letsencrypt/jobs/certbot-cleanup.py | 151 +-- .../core/letsencrypt/jobs/certbot-deploy.py | 210 ++-- .../core/letsencrypt/jobs/certbot-new.py | 934 +++++++----------- .../core/letsencrypt/jobs/certbot-renew.py | 225 ++--- .../core/letsencrypt/jobs/letsencrypt.py | 329 +++--- src/common/core/letsencrypt/letsencrypt.lua | 602 ++++++----- src/common/core/letsencrypt/ui/actions.py | 466 ++++----- .../letsencrypt/ui/blueprints/letsencrypt.py | 369 +++---- src/common/core/letsencrypt/ui/hooks.py | 68 +- 10 files changed, 1592 insertions(+), 1930 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-auth.py b/src/common/core/letsencrypt/jobs/certbot-auth.py index c3b28b5afd..be0f8532c9 100644 --- a/src/common/core/letsencrypt/jobs/certbot-auth.py +++ b/src/common/core/letsencrypt/jobs/certbot-auth.py @@ -20,6 +20,14 @@ LOGGER = setup_logger("Lets-encrypt.auth") status = 0 + +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + try: # Get environment variables for ACME HTTP challenge # CERTBOT_TOKEN: The filename for the challenge file @@ -27,21 +35,21 @@ token = getenv("CERTBOT_TOKEN", "") validation = getenv("CERTBOT_VALIDATION", "") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("ACME HTTP challenge authentication started") - LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") - LOGGER.debug(f"Validation length: {len(validation) if validation else 0} chars") - LOGGER.debug("Checking for required environment variables") - LOGGER.debug(f"CERTBOT_TOKEN exists: {bool(token)}") - LOGGER.debug(f"CERTBOT_VALIDATION exists: {bool(validation)}") + debug_log(LOGGER, "ACME HTTP challenge authentication started") + debug_log(LOGGER, f"Token: {token[:10] if token else 'None'}...") + debug_log(LOGGER, + f"Validation length: {len(validation) if validation else 0} chars") + debug_log(LOGGER, "Checking for required environment variables") + debug_log(LOGGER, f"CERTBOT_TOKEN exists: {bool(token)}") + debug_log(LOGGER, f"CERTBOT_VALIDATION exists: {bool(validation)}") # Detect the current BunkerWeb integration type # This determines how we handle the challenge deployment integration = get_integration() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Integration detection completed: {integration}") - LOGGER.debug("Determining challenge deployment method based on integration") + debug_log(LOGGER, f"Integration detection completed: {integration}") + debug_log(LOGGER, + "Determining challenge deployment method based on integration") LOGGER.info(f"Detected {integration} integration") @@ -49,9 +57,10 @@ # For cluster deployments, we need to distribute the challenge # to all instances via the BunkerWeb API if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Cluster integration detected, initializing database connection") - LOGGER.debug("Will distribute challenge to all cluster instances via API") + debug_log(LOGGER, + "Cluster integration detected, initializing database connection") + debug_log(LOGGER, + "Will distribute challenge to all cluster instances via API") # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -59,26 +68,26 @@ # Get all active BunkerWeb instances from the database instances = db.get_instances() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Retrieved {len(instances)} instances from database") - LOGGER.debug("Instance details:") - for i, instance in enumerate(instances): - LOGGER.debug( - f" Instance {i+1}: {instance['hostname']}:" - f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" - ) - LOGGER.debug("Preparing to send challenge data to each instance") + debug_log(LOGGER, f"Retrieved {len(instances)} instances from database") + debug_log(LOGGER, "Instance details:") + for i, instance in enumerate(instances): + debug_log(LOGGER, + f" Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: " + f"{instance.get('server_name', 'N/A')})") + debug_log(LOGGER, + "Preparing to send challenge data to each instance") LOGGER.info(f"Sending challenge to {len(instances)} instances") # Send challenge to each instance via API for instance in instances: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Processing instance: {instance['hostname']}:{instance['port']}" - ) - LOGGER.debug(f"Server name: {instance.get('server_name', 'N/A')}") - LOGGER.debug("Creating API client for this instance") + debug_log(LOGGER, + f"Processing instance: {instance['hostname']}:" + f"{instance['port']}") + debug_log(LOGGER, + f"Server name: {instance.get('server_name', 'N/A')}") + debug_log(LOGGER, "Creating API client for this instance") # Create API client for this instance api = API( @@ -86,11 +95,11 @@ host=instance["server_name"] ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API endpoint: {api.endpoint}") - LOGGER.debug("Preparing challenge data payload") - LOGGER.debug(f"Token: {token[:10]}... (truncated)") - LOGGER.debug(f"Validation length: {len(validation)} characters") + debug_log(LOGGER, f"API endpoint: {api.endpoint}") + debug_log(LOGGER, "Preparing challenge data payload") + debug_log(LOGGER, f"Token: {token[:10]}... (truncated)") + debug_log(LOGGER, + f"Validation length: {len(validation)} characters") # Send POST request to deploy the challenge sent, err, status_code, resp = api.request( @@ -105,9 +114,9 @@ f"Can't send API request to " f"{api.endpoint}/lets-encrypt/challenge: {err}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API request failed with error: {err}") - LOGGER.debug("This instance will not receive the challenge") + debug_log(LOGGER, f"API request failed with error: {err}") + debug_log(LOGGER, + "This instance will not receive the challenge") elif status_code != 200: status = 1 LOGGER.error( @@ -115,28 +124,31 @@ f"{api.endpoint}/lets-encrypt/challenge: " f"status = {resp['status']}, msg = {resp['msg']}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug(f"Response details: {resp}") - LOGGER.debug("Challenge deployment failed for this instance") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, f"Response details: {resp}") + debug_log(LOGGER, + "Challenge deployment failed for this instance") else: LOGGER.info( f"Successfully sent API request to " f"{api.endpoint}/lets-encrypt/challenge" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug("Challenge successfully deployed to instance") - LOGGER.debug(f"Instance can now serve challenge at /.well-known/acme-challenge/{token}") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, + "Challenge successfully deployed to instance") + debug_log(LOGGER, + f"Instance can now serve challenge at " + f"/.well-known/acme-challenge/{token}") # Linux case: Standalone installation # For standalone Linux installations, we write the challenge # file directly to the local filesystem else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Standalone Linux integration detected") - LOGGER.debug("Writing challenge file directly to local filesystem") - LOGGER.debug("No API distribution needed for standalone mode") + debug_log(LOGGER, "Standalone Linux integration detected") + debug_log(LOGGER, + "Writing challenge file directly to local filesystem") + debug_log(LOGGER, + "No API distribution needed for standalone mode") # Create the ACME challenge directory structure # This follows the standard .well-known/acme-challenge path @@ -145,35 +157,35 @@ ".well-known", "acme-challenge" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Challenge directory path: {root_dir}") - LOGGER.debug("Creating directory structure if it doesn't exist") - LOGGER.debug("Directory will be created with parents=True, exist_ok=True") + debug_log(LOGGER, f"Challenge directory path: {root_dir}") + debug_log(LOGGER, + "Creating directory structure if it doesn't exist") + debug_log(LOGGER, + "Directory will be created with parents=True, exist_ok=True") # Create directory structure with appropriate permissions root_dir.mkdir(parents=True, exist_ok=True) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Directory structure created successfully") - LOGGER.debug(f"Directory exists: {root_dir.exists()}") - LOGGER.debug(f"Directory is writable: {root_dir.is_dir()}") + debug_log(LOGGER, "Directory structure created successfully") + debug_log(LOGGER, f"Directory exists: {root_dir.exists()}") + debug_log(LOGGER, f"Directory is writable: {root_dir.is_dir()}") # Write the challenge validation content to the token file challenge_file = root_dir.joinpath(token) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Challenge file path: {challenge_file}") - LOGGER.debug(f"Token filename: {token}") - LOGGER.debug(f"Validation content length: {len(validation)} bytes") - LOGGER.debug("Writing validation content to challenge file") + debug_log(LOGGER, f"Challenge file path: {challenge_file}") + debug_log(LOGGER, f"Token filename: {token}") + debug_log(LOGGER, + f"Validation content length: {len(validation)} bytes") + debug_log(LOGGER, "Writing validation content to challenge file") challenge_file.write_text(validation, encoding="utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Challenge file written successfully") - LOGGER.debug(f"File exists: {challenge_file.exists()}") - LOGGER.debug(f"File size: {challenge_file.stat().st_size} bytes") - LOGGER.debug("Let's Encrypt can now access the challenge file") + debug_log(LOGGER, "Challenge file written successfully") + debug_log(LOGGER, f"File exists: {challenge_file.exists()}") + debug_log(LOGGER, + f"File size: {challenge_file.stat().st_size} bytes") + debug_log(LOGGER, "Let's Encrypt can now access the challenge file") LOGGER.info(f"Challenge file created at {challenge_file}") @@ -182,17 +194,17 @@ LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-auth.py:\n{e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Full exception traceback logged above") - LOGGER.debug("Authentication process failed due to exception") - LOGGER.debug("Let's Encrypt challenge will not be available") - -if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ACME HTTP challenge authentication completed with status: {status}") - if status == 0: - LOGGER.debug("Authentication completed successfully") - LOGGER.debug("Let's Encrypt can now access the challenge") - else: - LOGGER.debug("Authentication failed - challenge may not be accessible") + debug_log(LOGGER, "Full exception traceback logged above") + debug_log(LOGGER, "Authentication process failed due to exception") + debug_log(LOGGER, "Let's Encrypt challenge will not be available") + +debug_log(LOGGER, + f"ACME HTTP challenge authentication completed with status: {status}") +if status == 0: + debug_log(LOGGER, "Authentication completed successfully") + debug_log(LOGGER, "Let's Encrypt can now access the challenge") +else: + debug_log(LOGGER, + "Authentication failed - challenge may not be accessible") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-cleanup.py b/src/common/core/letsencrypt/jobs/certbot-cleanup.py index 3f6716dbff..6add17f139 100644 --- a/src/common/core/letsencrypt/jobs/certbot-cleanup.py +++ b/src/common/core/letsencrypt/jobs/certbot-cleanup.py @@ -20,24 +20,33 @@ LOGGER = setup_logger("Lets-encrypt.cleanup") status = 0 + +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + try: # Get environment variables for ACME HTTP challenge cleanup # CERTBOT_TOKEN: The filename of the challenge file to remove token = getenv("CERTBOT_TOKEN", "") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("ACME HTTP challenge cleanup started") - LOGGER.debug(f"Token to clean: {token[:10] if token else 'None'}...") - LOGGER.debug("Starting cleanup process for Let's Encrypt challenge") - LOGGER.debug("This will remove challenge files from all instances") + debug_log(LOGGER, "ACME HTTP challenge cleanup started") + debug_log(LOGGER, f"Token to clean: {token[:10] if token else 'None'}...") + debug_log(LOGGER, + "Starting cleanup process for Let's Encrypt challenge") + debug_log(LOGGER, + "This will remove challenge files from all instances") # Detect the current BunkerWeb integration type # This determines how we handle the challenge cleanup process integration = get_integration() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Integration detection completed: {integration}") - LOGGER.debug("Determining cleanup method based on integration type") + debug_log(LOGGER, f"Integration detection completed: {integration}") + debug_log(LOGGER, + "Determining cleanup method based on integration type") LOGGER.info(f"Detected {integration} integration") @@ -45,10 +54,11 @@ # For cluster deployments, we need to remove the challenge # from all instances via the BunkerWeb API if integration in ("Docker", "Swarm", "Kubernetes", "Autoconf"): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Cluster integration detected") - LOGGER.debug("Will remove challenge from all cluster instances via API") - LOGGER.debug("Initializing database connection to get instance list") + debug_log(LOGGER, "Cluster integration detected") + debug_log(LOGGER, + "Will remove challenge from all cluster instances via API") + debug_log(LOGGER, + "Initializing database connection to get instance list") # Initialize database connection to get list of instances db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -56,27 +66,26 @@ # Get all active BunkerWeb instances from the database instances = db.get_instances() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Retrieved {len(instances)} instances from database") - LOGGER.debug("Instance details for cleanup:") - for i, instance in enumerate(instances): - LOGGER.debug( - f" Instance {i+1}: {instance['hostname']}:" - f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" - ) - LOGGER.debug("Preparing to send DELETE requests to each instance") + debug_log(LOGGER, f"Retrieved {len(instances)} instances from database") + debug_log(LOGGER, "Instance details for cleanup:") + for i, instance in enumerate(instances): + debug_log(LOGGER, + f" Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: " + f"{instance.get('server_name', 'N/A')})") + debug_log(LOGGER, + "Preparing to send DELETE requests to each instance") LOGGER.info(f"Cleaning challenge from {len(instances)} instances") # Remove challenge from each instance via API for instance in instances: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Processing cleanup for instance: " - f"{instance['hostname']}:{instance['port']}" - ) - LOGGER.debug(f"Server name: {instance.get('server_name', 'N/A')}") - LOGGER.debug("Creating API client for cleanup request") + debug_log(LOGGER, + f"Processing cleanup for instance: " + f"{instance['hostname']}:{instance['port']}") + debug_log(LOGGER, + f"Server name: {instance.get('server_name', 'N/A')}") + debug_log(LOGGER, "Creating API client for cleanup request") # Create API client for this instance api = API( @@ -84,10 +93,9 @@ host=instance["server_name"] ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API endpoint: {api.endpoint}") - LOGGER.debug("Preparing DELETE request for challenge cleanup") - LOGGER.debug(f"Token to delete: {token}") + debug_log(LOGGER, f"API endpoint: {api.endpoint}") + debug_log(LOGGER, "Preparing DELETE request for challenge cleanup") + debug_log(LOGGER, f"Token to delete: {token}") # Send DELETE request to remove the challenge sent, err, status_code, resp = api.request( @@ -102,9 +110,9 @@ f"Can't send API request to " f"{api.endpoint}/lets-encrypt/challenge: {err}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"DELETE request failed with error: {err}") - LOGGER.debug("Challenge file may remain on this instance") + debug_log(LOGGER, f"DELETE request failed with error: {err}") + debug_log(LOGGER, + "Challenge file may remain on this instance") elif status_code != 200: status = 1 LOGGER.error( @@ -112,28 +120,28 @@ f"{api.endpoint}/lets-encrypt/challenge: " f"status = {resp['status']}, msg = {resp['msg']}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug(f"Response details: {resp}") - LOGGER.debug("Challenge cleanup failed for this instance") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, f"Response details: {resp}") + debug_log(LOGGER, + "Challenge cleanup failed for this instance") else: LOGGER.info( f"Successfully sent API request to " f"{api.endpoint}/lets-encrypt/challenge" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug("Challenge successfully removed from instance") - LOGGER.debug(f"Token {token} has been cleaned up") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, + "Challenge successfully removed from instance") + debug_log(LOGGER, f"Token {token} has been cleaned up") # Linux case: Standalone installation # For standalone Linux installations, we remove the challenge # file directly from the local filesystem else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Standalone Linux integration detected") - LOGGER.debug("Removing challenge file directly from local filesystem") - LOGGER.debug("No API cleanup needed for standalone mode") + debug_log(LOGGER, "Standalone Linux integration detected") + debug_log(LOGGER, + "Removing challenge file directly from local filesystem") + debug_log(LOGGER, "No API cleanup needed for standalone mode") # Construct path to the ACME challenge file # This follows the standard .well-known/acme-challenge path @@ -142,23 +150,26 @@ ".well-known", "acme-challenge", token ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Challenge file path: {challenge_file}") - LOGGER.debug(f"Token filename: {token}") - LOGGER.debug("Checking if challenge file exists before cleanup") - LOGGER.debug(f"File exists before cleanup: {challenge_file.exists()}") - if challenge_file.exists(): - LOGGER.debug(f"File size: {challenge_file.stat().st_size} bytes") + debug_log(LOGGER, f"Challenge file path: {challenge_file}") + debug_log(LOGGER, f"Token filename: {token}") + debug_log(LOGGER, + "Checking if challenge file exists before cleanup") + debug_log(LOGGER, + f"File exists before cleanup: {challenge_file.exists()}") + if challenge_file.exists(): + debug_log(LOGGER, + f"File size: {challenge_file.stat().st_size} bytes") # Remove the challenge file if it exists # missing_ok=True prevents errors if file doesn't exist challenge_file.unlink(missing_ok=True) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Challenge file unlink operation completed") - LOGGER.debug(f"File exists after cleanup: {challenge_file.exists()}") - LOGGER.debug("Local challenge file cleanup completed successfully") - LOGGER.debug("Challenge is no longer accessible to Let's Encrypt") + debug_log(LOGGER, "Challenge file unlink operation completed") + debug_log(LOGGER, + f"File exists after cleanup: {challenge_file.exists()}") + debug_log(LOGGER, "Local challenge file cleanup completed successfully") + debug_log(LOGGER, + "Challenge is no longer accessible to Let's Encrypt") LOGGER.info(f"Challenge file removed: {challenge_file}") @@ -167,17 +178,17 @@ LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-cleanup.py:\n{e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Full exception traceback logged above") - LOGGER.debug("Cleanup process failed due to exception") - LOGGER.debug("Some challenge files may remain uncleaned") + debug_log(LOGGER, "Full exception traceback logged above") + debug_log(LOGGER, "Cleanup process failed due to exception") + debug_log(LOGGER, "Some challenge files may remain uncleaned") -if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ACME HTTP challenge cleanup completed with status: {status}") - if status == 0: - LOGGER.debug("Cleanup completed successfully") - LOGGER.debug("All challenge files have been removed") - else: - LOGGER.debug("Cleanup encountered errors - some files may remain") +debug_log(LOGGER, + f"ACME HTTP challenge cleanup completed with status: {status}") +if status == 0: + debug_log(LOGGER, "Cleanup completed successfully") + debug_log(LOGGER, "All challenge files have been removed") +else: + debug_log(LOGGER, + "Cleanup encountered errors - some files may remain") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-deploy.py b/src/common/core/letsencrypt/jobs/certbot-deploy.py index c5eaf7c9a2..9f6915fae5 100644 --- a/src/common/core/letsencrypt/jobs/certbot-deploy.py +++ b/src/common/core/letsencrypt/jobs/certbot-deploy.py @@ -21,6 +21,14 @@ LOGGER = setup_logger("Lets-encrypt.deploy") status = 0 + +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + try: # Get environment variables for certificate deployment # CERTBOT_TOKEN: Token from certbot (currently unused but preserved) @@ -28,32 +36,33 @@ token = getenv("CERTBOT_TOKEN", "") renewed_domains = getenv("RENEWED_DOMAINS", "") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate deployment started") - LOGGER.debug(f"Token: {token[:10] if token else 'None'}...") - LOGGER.debug(f"Renewed domains: {renewed_domains}") - LOGGER.debug("Starting certificate distribution to cluster instances") - LOGGER.debug("This process will update certificates on all instances") + debug_log(LOGGER, "Certificate deployment started") + debug_log(LOGGER, f"Token: {token[:10] if token else 'None'}...") + debug_log(LOGGER, f"Renewed domains: {renewed_domains}") + debug_log(LOGGER, + "Starting certificate distribution to cluster instances") + debug_log(LOGGER, + "This process will update certificates on all instances") LOGGER.info(f"Certificates renewal for {renewed_domains} successful") # Create tarball of certificate directory for distribution # This packages all certificate files into a compressed archive # for efficient transfer to cluster instances - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Creating certificate archive for distribution") - LOGGER.debug("Packaging all Let's Encrypt certificates into tarball") - LOGGER.debug("Archive will contain fullchain.pem and privkey.pem files") + debug_log(LOGGER, "Creating certificate archive for distribution") + debug_log(LOGGER, + "Packaging all Let's Encrypt certificates into tarball") + debug_log(LOGGER, + "Archive will contain fullchain.pem and privkey.pem files") tgz = BytesIO() cert_source_path = join(sep, "var", "cache", "bunkerweb", "letsencrypt", "etc") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Source certificate path: {cert_source_path}") - LOGGER.debug("Checking if certificate directory exists") - cert_path_exists = Path(cert_source_path).exists() - LOGGER.debug(f"Certificate directory exists: {cert_path_exists}") + debug_log(LOGGER, f"Source certificate path: {cert_source_path}") + debug_log(LOGGER, "Checking if certificate directory exists") + cert_path_exists = Path(cert_source_path).exists() + debug_log(LOGGER, f"Certificate directory exists: {cert_path_exists}") # Create compressed tarball containing certificate files # compresslevel=3 provides good compression with reasonable performance @@ -64,17 +73,16 @@ tgz.seek(0, 0) files = {"archive.tar.gz": tgz} - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - archive_size = tgz.getbuffer().nbytes - LOGGER.debug(f"Certificate archive created successfully") - LOGGER.debug(f"Archive size: {archive_size} bytes") - LOGGER.debug(f"Compression level: 3 (balanced speed/size)") - LOGGER.debug("Archive ready for distribution to instances") + archive_size = tgz.getbuffer().nbytes + debug_log(LOGGER, "Certificate archive created successfully") + debug_log(LOGGER, f"Archive size: {archive_size} bytes") + debug_log(LOGGER, "Compression level: 3 (balanced speed/size)") + debug_log(LOGGER, "Archive ready for distribution to instances") # Initialize database connection to get cluster instances - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Initializing database connection") - LOGGER.debug("Need to get list of active BunkerWeb instances") + debug_log(LOGGER, "Initializing database connection") + debug_log(LOGGER, + "Need to get list of active BunkerWeb instances") db = Database(LOGGER, sqlalchemy_string=getenv("DATABASE_URI")) @@ -90,16 +98,17 @@ filtered_settings=("SERVER_NAME",) )["SERVER_NAME"].split(" ") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Retrieved {len(instances)} instances from database") - LOGGER.debug(f"Found {len(services)} configured services: {services}") - LOGGER.debug("Instance details for certificate deployment:") - for i, instance in enumerate(instances): - LOGGER.debug( - f" Instance {i+1}: {instance['hostname']}:" - f"{instance['port']} (server: {instance.get('server_name', 'N/A')})" - ) - LOGGER.debug("Will deploy certificates and trigger reload on each instance") + debug_log(LOGGER, f"Retrieved {len(instances)} instances from database") + debug_log(LOGGER, + f"Found {len(services)} configured services: {services}") + debug_log(LOGGER, "Instance details for certificate deployment:") + for i, instance in enumerate(instances): + debug_log(LOGGER, + f" Instance {i+1}: {instance['hostname']}:" + f"{instance['port']} (server: " + f"{instance.get('server_name', 'N/A')})") + debug_log(LOGGER, + "Will deploy certificates and trigger reload on each instance") # Configure reload timeout based on environment and service count # Minimum timeout prevents premature timeouts on slow systems @@ -114,37 +123,38 @@ # Calculate actual timeout: minimum timeout or 3 seconds per service calculated_timeout = max(reload_min_timeout, 3 * len(services)) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Reload timeout configuration:") - LOGGER.debug(f" Minimum timeout setting: {reload_min_timeout}s") - LOGGER.debug(f" Number of services: {len(services)}") - LOGGER.debug(f" Calculated timeout (max of min or 3s per service): {calculated_timeout}s") - LOGGER.debug("Timeout ensures all services have time to reload certificates") + debug_log(LOGGER, "Reload timeout configuration:") + debug_log(LOGGER, f" Minimum timeout setting: {reload_min_timeout}s") + debug_log(LOGGER, f" Number of services: {len(services)}") + debug_log(LOGGER, + f" Calculated timeout (max of min or 3s per service): " + f"{calculated_timeout}s") + debug_log(LOGGER, + "Timeout ensures all services have time to reload certificates") LOGGER.info(f"Deploying certificates to {len(instances)} instances") # Deploy certificates to each cluster instance for i, instance in enumerate(instances): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing instance {i+1}/{len(instances)}") - LOGGER.debug(f"Current instance: {instance['hostname']}:{instance['port']}") + debug_log(LOGGER, f"Processing instance {i+1}/{len(instances)}") + debug_log(LOGGER, + f"Current instance: {instance['hostname']}:{instance['port']}") # Construct API endpoint for this instance endpoint = f"http://{instance['hostname']}:{instance['port']}" host = instance["server_name"] - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"API endpoint: {endpoint}") - LOGGER.debug(f"Host header: {host}") - LOGGER.debug("Creating API client for certificate upload") + debug_log(LOGGER, f"API endpoint: {endpoint}") + debug_log(LOGGER, f"Host header: {host}") + debug_log(LOGGER, "Creating API client for certificate upload") api = API(endpoint, host=host) # Upload certificate archive to the instance - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Uploading certificate archive to {endpoint}") - LOGGER.debug("Sending POST request with certificate tarball") - LOGGER.debug("This will extract certificates to instance filesystem") + debug_log(LOGGER, f"Uploading certificate archive to {endpoint}") + debug_log(LOGGER, "Sending POST request with certificate tarball") + debug_log(LOGGER, + "This will extract certificates to instance filesystem") sent, err, status_code, resp = api.request( "POST", @@ -158,9 +168,9 @@ f"Can't send API request to " f"{api.endpoint}/lets-encrypt/certificates: {err}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate upload failed with error: {err}") - LOGGER.debug("Skipping reload for this instance due to upload failure") + debug_log(LOGGER, f"Certificate upload failed with error: {err}") + debug_log(LOGGER, + "Skipping reload for this instance due to upload failure") continue elif status_code != 200: status = 1 @@ -169,20 +179,19 @@ f"{api.endpoint}/lets-encrypt/certificates: " f"status = {resp['status']}, msg = {resp['msg']}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug(f"Error response: {resp}") - LOGGER.debug("Certificate upload failed, skipping reload") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, f"Error response: {resp}") + debug_log(LOGGER, "Certificate upload failed, skipping reload") continue else: LOGGER.info( f"Successfully sent API request to " f"{api.endpoint}/lets-encrypt/certificates" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate archive uploaded successfully") - LOGGER.debug("Certificates are now available on instance filesystem") - LOGGER.debug("Proceeding to trigger configuration reload") + debug_log(LOGGER, "Certificate archive uploaded successfully") + debug_log(LOGGER, + "Certificates are now available on instance filesystem") + debug_log(LOGGER, "Proceeding to trigger configuration reload") # Trigger configuration reload on the instance # Configuration testing can be disabled via environment variable @@ -190,11 +199,10 @@ "no").lower() test_config = "no" if disable_testing == "yes" else "yes" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Triggering configuration reload on {endpoint}") - LOGGER.debug(f"Configuration testing enabled: {test_config}") - LOGGER.debug(f"Reload timeout: {calculated_timeout}s") - LOGGER.debug("This will reload nginx with new certificates") + debug_log(LOGGER, f"Triggering configuration reload on {endpoint}") + debug_log(LOGGER, f"Configuration testing enabled: {test_config}") + debug_log(LOGGER, f"Reload timeout: {calculated_timeout}s") + debug_log(LOGGER, "This will reload nginx with new certificates") sent, err, status_code, resp = api.request( "POST", @@ -207,55 +215,59 @@ LOGGER.error( f"Can't send API request to {api.endpoint}/reload: {err}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Reload request failed with error: {err}") - LOGGER.debug("Instance may not have reloaded new certificates") + debug_log(LOGGER, f"Reload request failed with error: {err}") + debug_log(LOGGER, + "Instance may not have reloaded new certificates") elif status_code != 200: status = 1 LOGGER.error( f"Error while sending API request to {api.endpoint}/reload: " f"status = {resp['status']}, msg = {resp['msg']}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"HTTP status code: {status_code}") - LOGGER.debug(f"Reload response: {resp}") - LOGGER.debug("Configuration reload failed on this instance") + debug_log(LOGGER, f"HTTP status code: {status_code}") + debug_log(LOGGER, f"Reload response: {resp}") + debug_log(LOGGER, + "Configuration reload failed on this instance") else: - LOGGER.info(f"Successfully sent API request to {api.endpoint}/reload") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Configuration reload completed successfully") - LOGGER.debug("New certificates are now active on this instance") - LOGGER.debug(f"Instance {endpoint} fully updated") + LOGGER.info( + f"Successfully sent API request to {api.endpoint}/reload") + debug_log(LOGGER, "Configuration reload completed successfully") + debug_log(LOGGER, + "New certificates are now active on this instance") + debug_log(LOGGER, f"Instance {endpoint} fully updated") # Reset file pointer for next instance - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Resetting archive buffer position for next instance") - LOGGER.debug("Archive will be re-read from beginning for next upload") + debug_log(LOGGER, + "Resetting archive buffer position for next instance") + debug_log(LOGGER, + "Archive will be re-read from beginning for next upload") tgz.seek(0, 0) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate deployment process completed") - LOGGER.debug("All instances have been processed") - if status == 0: - LOGGER.debug("All deployments successful") - else: - LOGGER.debug("Some deployments failed - check individual instance logs") + debug_log(LOGGER, "Certificate deployment process completed") + debug_log(LOGGER, "All instances have been processed") + if status == 0: + debug_log(LOGGER, "All deployments successful") + else: + debug_log(LOGGER, + "Some deployments failed - check individual instance logs") except BaseException as e: status = 1 LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-deploy.py:\n{e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Full exception traceback logged above") - LOGGER.debug("Certificate deployment failed due to exception") - LOGGER.debug("Some instances may not have received updated certificates") + debug_log(LOGGER, "Full exception traceback logged above") + debug_log(LOGGER, "Certificate deployment failed due to exception") + debug_log(LOGGER, + "Some instances may not have received updated certificates") -if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate deployment completed with final status: {status}") - if status == 0: - LOGGER.debug("All certificates deployed successfully across cluster") - else: - LOGGER.debug("Deployment completed with errors - manual intervention may be needed") +debug_log(LOGGER, + f"Certificate deployment completed with final status: {status}") +if status == 0: + debug_log(LOGGER, + "All certificates deployed successfully across cluster") +else: + debug_log(LOGGER, + "Deployment completed with errors - manual intervention may be needed") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index c89737447c..d8c049f1b2 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -80,19 +80,19 @@ ) +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + def load_public_suffix_list(job): # Load and cache the public suffix list for domain validation. # Fetches the PSL from the official source and caches it locally. # Returns cached version if available and fresh (less than 1 day old). - # - # Args: - # job: Job instance for caching operations - # - # Returns: - # list: Lines from the public suffix list file - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Loading public suffix list from cache or {PSL_URL}") - LOGGER.debug("Checking if cached PSL is available and fresh") + debug_log(LOGGER, f"Loading public suffix list from cache or {PSL_URL}") + debug_log(LOGGER, "Checking if cached PSL is available and fresh") job_cache = job.get_cache(PSL_STATIC_FILE.name, with_info=True, with_data=True) @@ -103,25 +103,22 @@ def load_public_suffix_list(job): datetime.now().astimezone() - timedelta(days=1) ).timestamp() ): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Using cached public suffix list") - cache_age_hours = ((datetime.now().astimezone().timestamp() - - job_cache['last_update']) / 3600) - LOGGER.debug(f"Cache age: {cache_age_hours:.1f} hours") + debug_log(LOGGER, "Using cached public suffix list") + cache_age_hours = ((datetime.now().astimezone().timestamp() - + job_cache['last_update']) / 3600) + debug_log(LOGGER, f"Cache age: {cache_age_hours:.1f} hours") return job_cache["data"].decode("utf-8").splitlines() try: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Downloading fresh PSL from {PSL_URL}") - LOGGER.debug("Cached PSL is missing or older than 1 day") + debug_log(LOGGER, f"Downloading fresh PSL from {PSL_URL}") + debug_log(LOGGER, "Cached PSL is missing or older than 1 day") resp = get(PSL_URL, timeout=5) resp.raise_for_status() content = resp.text - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Downloaded PSL successfully, {len(content)} bytes") - LOGGER.debug(f"PSL contains {len(content.splitlines())} lines") + debug_log(LOGGER, f"Downloaded PSL successfully, {len(content)} bytes") + debug_log(LOGGER, f"PSL contains {len(content.splitlines())} lines") cached, err = JOB.cache_file(PSL_STATIC_FILE.name, content.encode("utf-8")) @@ -129,26 +126,22 @@ def load_public_suffix_list(job): LOGGER.error(f"Error while saving public suffix list to cache: " f"{err}") else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("PSL successfully cached for future use") + debug_log(LOGGER, "PSL successfully cached for future use") return content.splitlines() except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while downloading public suffix list: {e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Download failed, checking for existing static file") + debug_log(LOGGER, "Download failed, checking for existing static file") if PSL_STATIC_FILE.exists(): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using existing static PSL file: " - f"{PSL_STATIC_FILE}") + debug_log(LOGGER, + f"Using existing static PSL file: {PSL_STATIC_FILE}") with PSL_STATIC_FILE.open("r", encoding="utf-8") as f: return f.read().splitlines() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No PSL data available - returning empty list") + debug_log(LOGGER, "No PSL data available - returning empty list") return [] @@ -156,15 +149,8 @@ def parse_psl(psl_lines): # Parse PSL lines into rules and exceptions sets. # Processes the public suffix list format, handling comments, # exceptions (lines starting with !), and regular rules. - # - # Args: - # psl_lines: List of lines from the PSL file - # - # Returns: - # dict: Contains 'rules' and 'exceptions' sets - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsing {len(psl_lines)} PSL lines") - LOGGER.debug("Processing rules, exceptions, and filtering comments") + debug_log(LOGGER, f"Parsing {len(psl_lines)} PSL lines") + debug_log(LOGGER, "Processing rules, exceptions, and filtering comments") rules = set() exceptions = set() @@ -184,12 +170,11 @@ def parse_psl(psl_lines): continue rules.add(line) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsed {len(rules)} rules and {len(exceptions)} " - f"exceptions") - LOGGER.debug(f"Skipped {comments_skipped} comments and " - f"{empty_lines_skipped} empty lines") - LOGGER.debug("PSL parsing completed successfully") + debug_log(LOGGER, f"Parsed {len(rules)} rules and {len(exceptions)} " + f"exceptions") + debug_log(LOGGER, f"Skipped {comments_skipped} comments and " + f"{empty_lines_skipped} empty lines") + debug_log(LOGGER, "PSL parsing completed successfully") return {"rules": rules, "exceptions": exceptions} @@ -198,74 +183,56 @@ def is_domain_blacklisted(domain, psl): # Check if domain is forbidden by PSL rules. # Validates whether a domain would be blacklisted according to the # Public Suffix List rules and exceptions. - # - # Args: - # domain: Domain name to check - # psl: Parsed PSL data (dict with 'rules' and 'exceptions') - # - # Returns: - # bool: True if domain is blacklisted domain = domain.lower().strip(".") labels = domain.split(".") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking domain {domain} against PSL rules") - LOGGER.debug(f"Domain has {len(labels)} labels: {labels}") - LOGGER.debug(f"PSL contains {len(psl['rules'])} rules and " - f"{len(psl['exceptions'])} exceptions") + debug_log(LOGGER, f"Checking domain {domain} against PSL rules") + debug_log(LOGGER, f"Domain has {len(labels)} labels: {labels}") + debug_log(LOGGER, f"PSL contains {len(psl['rules'])} rules and " + f"{len(psl['exceptions'])} exceptions") for i in range(len(labels)): candidate = ".".join(str(label) for label in labels[i:]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking candidate: {candidate}") + debug_log(LOGGER, f"Checking candidate: {candidate}") if candidate in psl["exceptions"]: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} allowed by PSL exception " - f"{candidate}") + debug_log(LOGGER, f"Domain {domain} allowed by PSL exception " + f"{candidate}") return False if candidate in psl["rules"]: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found PSL rule match: {candidate}") - LOGGER.debug(f"Checking blacklist conditions for i={i}") + debug_log(LOGGER, f"Found PSL rule match: {candidate}") + debug_log(LOGGER, f"Checking blacklist conditions for i={i}") if i == 0: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - exact PSL " - f"rule match") + debug_log(LOGGER, f"Domain {domain} blacklisted - exact PSL " + f"rule match") return True if i == 0 and domain.startswith("*."): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Wildcard domain {domain} blacklisted - " - f"exact PSL rule match") + debug_log(LOGGER, f"Wildcard domain {domain} blacklisted - " + f"exact PSL rule match") return True if i == 0 or (i == 1 and labels[0] == "*"): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - PSL rule " - f"violation") + debug_log(LOGGER, f"Domain {domain} blacklisted - PSL rule " + f"violation") return True if len(labels[i:]) == len(labels): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - full label " - f"match") + debug_log(LOGGER, f"Domain {domain} blacklisted - full label " + f"match") return True wildcard_candidate = f"*.{candidate}" if wildcard_candidate in psl["rules"]: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found PSL wildcard rule match: " - f"{wildcard_candidate}") + debug_log(LOGGER, f"Found PSL wildcard rule match: " + f"{wildcard_candidate}") if len(labels[i:]) == 2: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} blacklisted - wildcard " - f"PSL rule match") + debug_log(LOGGER, f"Domain {domain} blacklisted - wildcard " + f"PSL rule match") return True - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {domain} not blacklisted by PSL") + debug_log(LOGGER, f"Domain {domain} not blacklisted by PSL") return False @@ -273,16 +240,8 @@ def get_certificate_authority_config(ca_provider, staging=False): # Get ACME server configuration for the specified CA provider. # Returns the appropriate ACME server URL and name for the given # certificate authority and environment (staging/production). - # - # Args: - # ca_provider: Certificate authority name ('zerossl' or 'letsencrypt') - # staging: Whether to use staging environment - # - # Returns: - # dict: Server URL and CA name - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Getting CA config for {ca_provider}, " - f"staging={staging}") + debug_log(LOGGER, f"Getting CA config for {ca_provider}, " + f"staging={staging}") if ca_provider.lower() == "zerossl": config = { @@ -297,8 +256,7 @@ def get_certificate_authority_config(ca_provider, staging=False): "name": "Let's Encrypt" } - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CA config: {config}") + debug_log(LOGGER, f"CA config: {config}") return config @@ -307,13 +265,6 @@ def setup_zerossl_eab_credentials(email, api_key=None): # Setup External Account Binding (EAB) credentials for ZeroSSL. # Contacts the ZeroSSL API to obtain EAB credentials required for # ACME certificate issuance with ZeroSSL. - # - # Args: - # email: Email address for the account - # api_key: ZeroSSL API key - # - # Returns: - # tuple: (eab_kid, eab_hmac_key) or (None, None) on failure LOGGER.info(f"Setting up ZeroSSL EAB credentials for email: {email}") if not api_key: @@ -323,18 +274,16 @@ def setup_zerossl_eab_credentials(email, api_key=None): ) return None, None - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Making request to ZeroSSL API for EAB credentials") - LOGGER.debug(f"Email: {email}") - LOGGER.debug(f"API key provided: {bool(api_key)}") + debug_log(LOGGER, "Making request to ZeroSSL API for EAB credentials") + debug_log(LOGGER, f"Email: {email}") + debug_log(LOGGER, f"API key provided: {bool(api_key)}") LOGGER.info("Making request to ZeroSSL API for EAB credentials") # Try the correct ZeroSSL API endpoint try: # The correct endpoint for ZeroSSL EAB credentials - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Attempting primary ZeroSSL EAB endpoint") + debug_log(LOGGER, "Attempting primary ZeroSSL EAB endpoint") response = get( "https://api.zerossl.com/acme/eab-credentials", @@ -342,18 +291,16 @@ def setup_zerossl_eab_credentials(email, api_key=None): timeout=30 ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL API response status: " - f"{response.status_code}") - LOGGER.debug(f"Response headers: {dict(response.headers)}") + debug_log(LOGGER, f"ZeroSSL API response status: " + f"{response.status_code}") + debug_log(LOGGER, f"Response headers: {dict(response.headers)}") LOGGER.info(f"ZeroSSL API response status: {response.status_code}") if response.status_code == 200: response.raise_for_status() eab_data = response.json() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL API response data: {eab_data}") + debug_log(LOGGER, f"ZeroSSL API response data: {eab_data}") LOGGER.info(f"ZeroSSL API response data: {eab_data}") # ZeroSSL typically returns eab_kid and eab_hmac_key directly @@ -380,13 +327,11 @@ def setup_zerossl_eab_credentials(email, api_key=None): ) response_text = response.text - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Primary endpoint response: {response_text}") + debug_log(LOGGER, f"Primary endpoint response: {response_text}") LOGGER.info(f"Primary endpoint response: {response_text}") # Try alternative endpoint with email parameter - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Attempting alternative ZeroSSL EAB endpoint") + debug_log(LOGGER, "Attempting alternative ZeroSSL EAB endpoint") response = get( "https://api.zerossl.com/acme/eab-credentials-email", @@ -395,18 +340,16 @@ def setup_zerossl_eab_credentials(email, api_key=None): timeout=30 ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - alt_status = response.status_code - LOGGER.debug(f"Alternative ZeroSSL API response status: " - f"{alt_status}") + alt_status = response.status_code + debug_log(LOGGER, f"Alternative ZeroSSL API response status: " + f"{alt_status}") LOGGER.info(f"Alternative ZeroSSL API response status: " f"{response.status_code}") response.raise_for_status() eab_data = response.json() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Alternative ZeroSSL API response data: " - f"{eab_data}") + debug_log(LOGGER, f"Alternative ZeroSSL API response data: " + f"{eab_data}") LOGGER.info(f"Alternative ZeroSSL API response data: {eab_data}") if eab_data.get("success"): @@ -430,8 +373,7 @@ def setup_zerossl_eab_credentials(email, api_key=None): LOGGER.debug(format_exc()) LOGGER.error(f"❌ Error setting up ZeroSSL EAB credentials: {e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("ZeroSSL EAB setup failed with exception") + debug_log(LOGGER, "ZeroSSL EAB setup failed with exception") # Additional troubleshooting info LOGGER.error("Troubleshooting steps:") @@ -447,25 +389,17 @@ def get_caa_records(domain): # Get CAA records for a domain using dig command. # Queries DNS CAA records to check certificate authority authorization. # Returns None if dig command is not available. - # - # Args: - # domain: Domain name to query - # - # Returns: - # list or None: List of CAA record dicts or None if unavailable # Check if dig command is available if not which("dig"): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("dig command not available for CAA record checking") + debug_log(LOGGER, "dig command not available for CAA record checking") LOGGER.info("dig command not available for CAA record checking") return None try: # Use dig to query CAA records - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Querying CAA records for domain: {domain}") - LOGGER.debug("Using dig command with +short flag") + debug_log(LOGGER, f"Querying CAA records for domain: {domain}") + debug_log(LOGGER, "Using dig command with +short flag") LOGGER.info(f"Querying CAA records for domain: {domain}") result = run( @@ -475,24 +409,21 @@ def get_caa_records(domain): timeout=10 ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"dig command return code: {result.returncode}") - LOGGER.debug(f"dig stdout: {result.stdout}") - LOGGER.debug(f"dig stderr: {result.stderr}") + debug_log(LOGGER, f"dig command return code: {result.returncode}") + debug_log(LOGGER, f"dig stdout: {result.stdout}") + debug_log(LOGGER, f"dig stderr: {result.stderr}") if result.returncode == 0 and result.stdout.strip(): LOGGER.info(f"Found CAA records for domain {domain}") caa_records = [] raw_lines = result.stdout.strip().split('\n') - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing {len(raw_lines)} CAA record lines") + debug_log(LOGGER, f"Processing {len(raw_lines)} CAA record lines") for line in raw_lines: line = line.strip() if line: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsing CAA record line: {line}") + debug_log(LOGGER, f"Parsing CAA record line: {line}") # CAA record format: flags tag "value" # Example: 0 issue "letsencrypt.org" @@ -507,23 +438,19 @@ def get_caa_records(domain): 'value': value }) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Parsed CAA record: flags={flags}, " - f"tag={tag}, value={value}") + debug_log(LOGGER, f"Parsed CAA record: flags={flags}, " + f"tag={tag}, value={value}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - record_count = len(caa_records) - LOGGER.debug(f"Successfully parsed {record_count} CAA records " - f"for domain {domain}") + record_count = len(caa_records) + debug_log(LOGGER, f"Successfully parsed {record_count} CAA records " + f"for domain {domain}") LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain " f"{domain}") return caa_records else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"No CAA records found for domain {domain} " - f"(dig return code: {result.returncode})" - ) + debug_log(LOGGER, + f"No CAA records found for domain {domain} " + f"(dig return code: {result.returncode})") LOGGER.info( f"No CAA records found for domain {domain} " f"(dig return code: {result.returncode})" @@ -531,8 +458,7 @@ def get_caa_records(domain): return [] except BaseException as e: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Error querying CAA records for {domain}: {e}") + debug_log(LOGGER, f"Error querying CAA records for {domain}: {e}") LOGGER.info(f"Error querying CAA records for {domain}: {e}") return None @@ -541,19 +467,9 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Check if the CA provider is authorized by CAA records. # Validates whether the certificate authority is permitted to issue # certificates for the domain according to CAA DNS records. - # - # Args: - # domain: Domain name to check - # ca_provider: Certificate authority provider name - # is_wildcard: Whether this is for a wildcard certificate - # - # Returns: - # bool: True if CA is authorized or no CAA restrictions exist - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Checking CAA authorization for domain: {domain}, " - f"CA: {ca_provider}, wildcard: {is_wildcard}" - ) + debug_log(LOGGER, + f"Checking CAA authorization for domain: {domain}, " + f"CA: {ca_provider}, wildcard: {is_wildcard}") LOGGER.info( f"Checking CAA authorization for domain: {domain}, " @@ -569,30 +485,26 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): allowed_identifiers = ca_identifiers.get(ca_provider.lower(), []) if not allowed_identifiers: LOGGER.warning(f"Unknown CA provider for CAA check: {ca_provider}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Returning True for unknown CA provider " - "(conservative approach)") + debug_log(LOGGER, "Returning True for unknown CA provider " + "(conservative approach)") return True # Allow unknown providers (conservative approach) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CA identifiers for {ca_provider}: " - f"{allowed_identifiers}") + debug_log(LOGGER, f"CA identifiers for {ca_provider}: " + f"{allowed_identifiers}") # Check CAA records for the domain and parent domains check_domain = domain.lstrip("*.") domain_parts = check_domain.split(".") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Will check CAA records for domain chain: " - f"{check_domain}") - LOGGER.debug(f"Domain parts: {domain_parts}") + debug_log(LOGGER, f"Will check CAA records for domain chain: " + f"{check_domain}") + debug_log(LOGGER, f"Domain parts: {domain_parts}") LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") for i in range(len(domain_parts)): current_domain = ".".join(str(part) for part in domain_parts[i:]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking CAA records for: {current_domain}") + debug_log(LOGGER, f"Checking CAA records for: {current_domain}") LOGGER.info(f"Checking CAA records for: {current_domain}") caa_records = get_caa_records(current_domain) @@ -600,8 +512,7 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # dig not available, skip CAA check LOGGER.info("CAA record checking skipped (dig command not " "available)") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Returning True due to unavailable dig command") + debug_log(LOGGER, "Returning True due to unavailable dig command") return True if caa_records: @@ -619,15 +530,13 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Log found records if issue_records: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA issue records: " - f"{', '.join(str(record) for record in issue_records)}") + debug_log(LOGGER, f"CAA issue records: " + f"{', '.join(str(record) for record in issue_records)}") LOGGER.info(f"CAA issue records: " f"{', '.join(str(record) for record in issue_records)}") if issuewild_records: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA issuewild records: " - f"{', '.join(str(record) for record in issuewild_records)}") + debug_log(LOGGER, f"CAA issuewild records: " + f"{', '.join(str(record) for record in issuewild_records)}") LOGGER.info(f"CAA issuewild records: " f"{', '.join(str(record) for record in issuewild_records)}") @@ -644,19 +553,16 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): check_records = issue_records record_type = "issue" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using CAA {record_type} records for " - f"authorization check") - LOGGER.debug(f"Records to check: {check_records}") + debug_log(LOGGER, f"Using CAA {record_type} records for " + f"authorization check") + debug_log(LOGGER, f"Records to check: {check_records}") LOGGER.info(f"Using CAA {record_type} records for authorization " f"check") if not check_records: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"No relevant CAA {record_type} records found for " - f"{current_domain}" - ) + debug_log(LOGGER, + f"No relevant CAA {record_type} records found for " + f"{current_domain}") LOGGER.info( f"No relevant CAA {record_type} records found for " f"{current_domain}" @@ -666,22 +572,18 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Check if any of our CA identifiers are authorized authorized = False - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - identifier_list = ', '.join(str(id) for id in allowed_identifiers) - LOGGER.debug( - f"Checking authorization for CA identifiers: " - f"{identifier_list}" - ) identifier_list = ', '.join(str(id) for id in allowed_identifiers) + debug_log(LOGGER, + f"Checking authorization for CA identifiers: " + f"{identifier_list}") LOGGER.info( f"Checking authorization for CA identifiers: " f"{identifier_list}" ) for identifier in allowed_identifiers: for record in check_records: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Comparing identifier '{identifier}' " - f"with record '{record}'") + debug_log(LOGGER, f"Comparing identifier '{identifier}' " + f"with record '{record}'") # Handle explicit deny (empty value or semicolon) if record == ";" or record.strip() == "": @@ -689,9 +591,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"CAA {record_type} record explicitly denies " f"all CAs" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Found explicit deny record - " - "authorization failed") + debug_log(LOGGER, "Found explicit deny record - " + "authorization failed") return False # Check for CA authorization @@ -701,9 +602,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"✓ CA {ca_provider} ({identifier}) authorized " f"by CAA {record_type} record" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Authorization found: {identifier} " - f"in {record}") + debug_log(LOGGER, f"Authorization found: {identifier} " + f"in {record}") break if authorized: break @@ -722,17 +622,15 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): LOGGER.error( f"But {ca_provider} uses: {identifier_list}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("CAA authorization failed - no matching " - "identifiers") + debug_log(LOGGER, "CAA authorization failed - no matching " + "identifiers") return False # If we found CAA records and we're authorized, we can stop # checking parent domains LOGGER.info(f"✓ CAA authorization successful for {domain}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("CAA authorization successful - stopping parent " - "domain checks") + debug_log(LOGGER, "CAA authorization successful - stopping parent " + "domain checks") return True # No CAA records found in the entire chain @@ -740,9 +638,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"No CAA records found for {check_domain} or parent domains - " f"any CA allowed" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No CAA records found in entire domain chain - " - "allowing any CA") + debug_log(LOGGER, "No CAA records found in entire domain chain - " + "allowing any CA") return True @@ -753,23 +650,12 @@ def validate_domains_for_http_challenge(domains_list, # for HTTP challenge. # Checks DNS resolution and certificate authority authorization for each # domain in the list to ensure HTTP challenge will succeed. - # - # Args: - # domains_list: List of domain names to validate - # ca_provider: Certificate authority provider name - # is_wildcard: Whether this is for wildcard certificates - # - # Returns: - # bool: True if all domains are valid for HTTP challenge - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - domain_count = len(domains_list) - domain_list = ', '.join(str(domain) for domain in domains_list) - LOGGER.debug( - f"Validating {domain_count} domains for HTTP challenge: " - f"{domain_list}" - ) - LOGGER.debug(f"CA provider: {ca_provider}, wildcard: {is_wildcard}") + domain_count = len(domains_list) domain_list = ', '.join(str(domain) for domain in domains_list) + debug_log(LOGGER, + f"Validating {domain_count} domains for HTTP challenge: " + f"{domain_list}") + debug_log(LOGGER, f"CA provider: {ca_provider}, wildcard: {is_wildcard}") LOGGER.info( f"Validating {len(domains_list)} domains for HTTP challenge: " f"{domain_list}" @@ -780,9 +666,8 @@ def validate_domains_for_http_challenge(domains_list, # Check if CAA validation should be skipped skip_caa_check = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - caa_status = 'skipped' if skip_caa_check else 'performed' - LOGGER.debug(f"CAA check will be {caa_status}") + caa_status = 'skipped' if skip_caa_check else 'performed' + debug_log(LOGGER, f"CAA check will be {caa_status}") # Get external IPs once for all domain checks external_ips = get_external_ip() @@ -803,15 +688,13 @@ def validate_domains_for_http_challenge(domains_list, validation_failed = 0 for domain in domains_list: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Validating domain: {domain}") + debug_log(LOGGER, f"Validating domain: {domain}") # Check DNS A/AAAA records with retry mechanism if not check_domain_a_record(domain, external_ips): invalid_domains.append(domain) validation_failed += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"DNS validation failed for {domain}") + debug_log(LOGGER, f"DNS validation failed for {domain}") continue # Check CAA authorization @@ -819,22 +702,18 @@ def validate_domains_for_http_challenge(domains_list, if not check_caa_authorization(domain, ca_provider, is_wildcard): caa_blocked_domains.append(domain) validation_failed += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA authorization failed for {domain}") + debug_log(LOGGER, f"CAA authorization failed for {domain}") else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CAA check skipped for {domain} " - f"(ACME_SKIP_CAA_CHECK=yes)") + debug_log(LOGGER, f"CAA check skipped for {domain} " + f"(ACME_SKIP_CAA_CHECK=yes)") LOGGER.info(f"CAA check skipped for {domain} " f"(ACME_SKIP_CAA_CHECK=yes)") validation_passed += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Validation passed for {domain}") + debug_log(LOGGER, f"Validation passed for {domain}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Validation summary: {validation_passed} passed, " - f"{validation_failed} failed") + debug_log(LOGGER, f"Validation summary: {validation_passed} passed, " + f"{validation_failed} failed") # Report results if invalid_domains: @@ -875,11 +754,7 @@ def get_external_ip(): # Get the external/public IP addresses of this server (both IPv4 and IPv6). # Queries multiple external services to determine the server's public # IP addresses for DNS validation purposes. - # - # Returns: - # dict or None: Dict with 'ipv4' and 'ipv6' keys, or None if all fail - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Getting external IP addresses for server") + debug_log(LOGGER, "Getting external IP addresses for server") LOGGER.info("Getting external IP addresses for server") ipv4_services = [ @@ -898,16 +773,14 @@ def get_external_ip(): external_ips: Dict[str, Optional[str]] = {"ipv4": None, "ipv6": None} # Try to get IPv4 address - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Attempting to get external IPv4 address") - LOGGER.debug(f"Trying {len(ipv4_services)} IPv4 services") + debug_log(LOGGER, "Attempting to get external IPv4 address") + debug_log(LOGGER, f"Trying {len(ipv4_services)} IPv4 services") LOGGER.info("Attempting to get external IPv4 address") for i, service in enumerate(ipv4_services): try: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - service_num = f"{i+1}/{len(ipv4_services)}" - LOGGER.debug(f"Trying IPv4 service {service_num}: {service}") + service_num = f"{i+1}/{len(ipv4_services)}" + debug_log(LOGGER, f"Trying IPv4 service {service_num}: {service}") if "jsonip.com" in service: # This service returns JSON format @@ -921,8 +794,7 @@ def get_external_ip(): response.raise_for_status() ip = response.text.strip() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Service returned: {ip}") + debug_log(LOGGER, f"Service returned: {ip}") # Basic IPv4 validation if ip and "." in ip and len(ip.split(".")) == 4: @@ -933,34 +805,29 @@ def get_external_ip(): ipv4_addr: str = str(ip) external_ips["ipv4"] = ipv4_addr - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Successfully obtained external IPv4 " - f"address: {ipv4_addr}") + debug_log(LOGGER, f"Successfully obtained external IPv4 " + f"address: {ipv4_addr}") LOGGER.info(f"Successfully obtained external IPv4 " f"address: {ipv4_addr}") break except gaierror: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Invalid IPv4 address returned: {ip}") + debug_log(LOGGER, f"Invalid IPv4 address returned: {ip}") continue except BaseException as e: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Failed to get IPv4 address from {service}: " - f"{e}") + debug_log(LOGGER, f"Failed to get IPv4 address from {service}: " + f"{e}") LOGGER.info(f"Failed to get IPv4 address from {service}: {e}") continue # Try to get IPv6 address - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Attempting to get external IPv6 address") - LOGGER.debug(f"Trying {len(ipv6_services)} IPv6 services") + debug_log(LOGGER, "Attempting to get external IPv6 address") + debug_log(LOGGER, f"Trying {len(ipv6_services)} IPv6 services") LOGGER.info("Attempting to get external IPv6 address") for i, service in enumerate(ipv6_services): try: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - service_num = f"{i+1}/{len(ipv6_services)}" - LOGGER.debug(f"Trying IPv6 service {service_num}: {service}") + service_num = f"{i+1}/{len(ipv6_services)}" + debug_log(LOGGER, f"Trying IPv6 service {service_num}: {service}") if "jsonip.com" in service: response = get(service, timeout=5) @@ -972,8 +839,7 @@ def get_external_ip(): response.raise_for_status() ip = response.text.strip() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Service returned: {ip}") + debug_log(LOGGER, f"Service returned: {ip}") # Basic IPv6 validation if ip and ":" in ip: @@ -984,20 +850,17 @@ def get_external_ip(): ipv6_addr: str = str(ip) external_ips["ipv6"] = ipv6_addr - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Successfully obtained external IPv6 " - f"address: {ipv6_addr}") + debug_log(LOGGER, f"Successfully obtained external IPv6 " + f"address: {ipv6_addr}") LOGGER.info(f"Successfully obtained external IPv6 " f"address: {ipv6_addr}") break except gaierror: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Invalid IPv6 address returned: {ip}") + debug_log(LOGGER, f"Invalid IPv6 address returned: {ip}") continue except BaseException as e: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Failed to get IPv6 address from {service}: " - f"{e}") + debug_log(LOGGER, f"Failed to get IPv6 address from {service}: " + f"{e}") LOGGER.info(f"Failed to get IPv6 address from {service}: {e}") continue @@ -1006,8 +869,7 @@ def get_external_ip(): "Could not determine external IP address (IPv4 or IPv6) from " "any service" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("All external IP services failed") + debug_log(LOGGER, "All external IP services failed") return None ipv4_status = external_ips['ipv4'] or 'not found' @@ -1023,23 +885,14 @@ def check_domain_a_record(domain, external_ips=None): # Check if domain has valid A/AAAA records for HTTP challenge. # Validates DNS resolution and optionally checks if the domain's # IP addresses match the server's external IPs. - # - # Args: - # domain: Domain name to check - # external_ips: Dict with server's external IPv4/IPv6 addresses - # - # Returns: - # bool: True if domain has valid DNS records - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking DNS A/AAAA records for domain: {domain}") + debug_log(LOGGER, f"Checking DNS A/AAAA records for domain: {domain}") LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") try: # Remove wildcard prefix if present check_domain = domain.lstrip("*.") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking domain after wildcard removal: " - f"{check_domain}") + debug_log(LOGGER, f"Checking domain after wildcard removal: " + f"{check_domain}") # Attempt to resolve the domain to IP addresses result = getaddrinfo(check_domain, None) @@ -1049,39 +902,31 @@ def check_domain_a_record(domain, external_ips=None): ipv6_addresses = [addr[4][0] for addr in result if addr[0] == AF_INET6] - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"DNS resolution results:") - LOGGER.debug(f" IPv4 addresses: {ipv4_addresses}") - LOGGER.debug(f" IPv6 addresses: {ipv6_addresses}") + debug_log(LOGGER, "DNS resolution results:") + debug_log(LOGGER, f" IPv4 addresses: {ipv4_addresses}") + debug_log(LOGGER, f" IPv6 addresses: {ipv6_addresses}") if not ipv4_addresses and not ipv6_addresses: LOGGER.warning(f"Domain {check_domain} has no A or AAAA " f"records") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No valid IP addresses found in DNS " - "resolution") + debug_log(LOGGER, "No valid IP addresses found in DNS " + "resolution") return False # Log found addresses if ipv4_addresses: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - ipv4_display = ', '.join(str(addr) for addr in ipv4_addresses[:3]) - LOGGER.debug( - f"Domain {check_domain} IPv4 A records: " - f"{ipv4_display}" - ) ipv4_display = ', '.join(str(addr) for addr in ipv4_addresses[:3]) + debug_log(LOGGER, + f"Domain {check_domain} IPv4 A records: " + f"{ipv4_display}") LOGGER.info( f"Domain {check_domain} IPv4 A records: {ipv4_display}" ) if ipv6_addresses: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - ipv6_display = ', '.join(str(addr) for addr in ipv6_addresses[:3]) - LOGGER.debug( - f"Domain {check_domain} IPv6 AAAA records: " - f"{ipv6_display}" - ) ipv6_display = ', '.join(str(addr) for addr in ipv6_addresses[:3]) + debug_log(LOGGER, + f"Domain {check_domain} IPv6 AAAA records: " + f"{ipv6_display}") LOGGER.info( f"Domain {check_domain} IPv6 AAAA records: " f"{ipv6_display}" @@ -1092,9 +937,8 @@ def check_domain_a_record(domain, external_ips=None): ipv4_match = False ipv6_match = False - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking IP address matches with server " - "external IPs") + debug_log(LOGGER, "Checking IP address matches with server " + "external IPs") # Check IPv4 match if external_ips.get("ipv4") and ipv4_addresses: @@ -1139,11 +983,10 @@ def check_domain_a_record(domain, external_ips=None): has_external_ip = (external_ips.get("ipv4") or external_ips.get("ipv6")) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"IP match results: IPv4={ipv4_match}, " - f"IPv6={ipv6_match}") - LOGGER.debug(f"Has external IP: {has_external_ip}, " - f"Has match: {has_any_match}") + debug_log(LOGGER, f"IP match results: IPv4={ipv4_match}, " + f"IPv6={ipv6_match}") + debug_log(LOGGER, f"Has external IP: {has_external_ip}, " + f"Has match: {has_any_match}") if has_external_ip and not has_any_match: LOGGER.warning( @@ -1163,30 +1006,25 @@ def check_domain_a_record(domain, external_ips=None): f"Strict IP check enabled - rejecting certificate " f"request for {check_domain}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Strict IP check failed - " - "returning False") + debug_log(LOGGER, "Strict IP check failed - " + "returning False") return False LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("DNS validation completed successfully") + debug_log(LOGGER, "DNS validation completed successfully") return True else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Domain {check_domain} validation failed - no DNS " - f"resolution" - ) + debug_log(LOGGER, + f"Domain {check_domain} validation failed - no DNS " + f"resolution") LOGGER.info(f"Domain {check_domain} validation failed - no DNS " f"resolution") LOGGER.warning(f"Domain {check_domain} does not resolve") return False except gaierror as e: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Domain {check_domain} DNS resolution failed " - f"(gaierror): {e}") + debug_log(LOGGER, f"Domain {check_domain} DNS resolution failed " + f"(gaierror): {e}") LOGGER.info(f"Domain {check_domain} DNS resolution failed " f"(gaierror): {e}") LOGGER.warning(f"DNS resolution failed for domain {check_domain}: " @@ -1196,8 +1034,7 @@ def check_domain_a_record(domain, external_ips=None): LOGGER.info(format_exc()) LOGGER.error(f"Error checking DNS records for domain " f"{check_domain}: {e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("DNS check failed with unexpected exception") + debug_log(LOGGER, "DNS check failed with unexpected exception") return False @@ -1220,29 +1057,9 @@ def certbot_new_with_retry( # Execute certbot with retry mechanism. # Wrapper around certbot_new that implements automatic retries with # exponential backoff for failed certificate generation attempts. - # - # Args: - # challenge_type: Type of ACME challenge ('dns' or 'http') - # domains: Comma-separated list of domain names - # email: Email address for certificate registration - # provider: DNS provider name (for DNS challenge) - # credentials_path: Path to credentials file - # propagation: DNS propagation time in seconds - # profile: Certificate profile to use - # staging: Whether to use staging environment - # force: Force renewal of existing certificates - # cmd_env: Environment variables for certbot process - # max_retries: Maximum number of retry attempts - # ca_provider: Certificate authority provider - # api_key: API key for CA (if required) - # server_name: Server name for multisite configurations - # - # Returns: - # int: Exit code (0 for success) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Starting certbot with retry for domains: {domains}") - LOGGER.debug(f"Max retries: {max_retries}, CA: {ca_provider}") - LOGGER.debug(f"Challenge: {challenge_type}, Provider: {provider}") + debug_log(LOGGER, f"Starting certbot with retry for domains: {domains}") + debug_log(LOGGER, f"Max retries: {max_retries}, CA: {ca_provider}") + debug_log(LOGGER, f"Challenge: {challenge_type}, Provider: {provider}") attempt = 1 while attempt <= max_retries + 1: @@ -1253,15 +1070,13 @@ def certbot_new_with_retry( ) wait_time = min(30 * (2 ** (attempt - 2)), 300) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Waiting {wait_time} seconds before retry...") - LOGGER.debug(f"Exponential backoff: base=30s, " - f"attempt={attempt}") + debug_log(LOGGER, f"Waiting {wait_time} seconds before retry...") + debug_log(LOGGER, f"Exponential backoff: base=30s, " + f"attempt={attempt}") LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Executing certbot attempt {attempt}") + debug_log(LOGGER, f"Executing certbot attempt {attempt}") result = certbot_new( challenge_type, @@ -1283,19 +1098,16 @@ def certbot_new_with_retry( if attempt > 1: LOGGER.info(f"Certificate generation succeeded on attempt " f"{attempt}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certbot completed successfully") + debug_log(LOGGER, "Certbot completed successfully") return result if attempt >= max_retries + 1: LOGGER.error(f"Certificate generation failed after " f"{max_retries + 1} attempts") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Maximum retries reached - giving up") + debug_log(LOGGER, "Maximum retries reached - giving up") return result - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Attempt {attempt} failed, will retry") + debug_log(LOGGER, f"Attempt {attempt} failed, will retry") attempt += 1 return result @@ -1319,34 +1131,15 @@ def certbot_new( # Generate new certificate using certbot. # Main function to request SSL/TLS certificates from a certificate authority # using the ACME protocol via certbot. - # - # Args: - # challenge_type: Type of ACME challenge ('dns' or 'http') - # domains: Comma-separated list of domain names - # email: Email address for certificate registration - # provider: DNS provider name (for DNS challenge) - # credentials_path: Path to credentials file - # propagation: DNS propagation time in seconds - # profile: Certificate profile to use - # staging: Whether to use staging environment - # force: Force renewal of existing certificates - # cmd_env: Environment variables for certbot process - # ca_provider: Certificate authority provider - # api_key: API key for CA (if required) - # server_name: Server name for multisite configurations - # - # Returns: - # int: Exit code (0 for success) if isinstance(credentials_path, str): credentials_path = Path(credentials_path) ca_config = get_certificate_authority_config(ca_provider, staging) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Building certbot command for {domains}") - LOGGER.debug(f"CA config: {ca_config}") - LOGGER.debug(f"Challenge type: {challenge_type}") - LOGGER.debug(f"Profile: {profile}") + debug_log(LOGGER, f"Building certbot command for {domains}") + debug_log(LOGGER, f"CA config: {ca_config}") + debug_log(LOGGER, f"Challenge type: {challenge_type}") + debug_log(LOGGER, f"Profile: {profile}") command = [ CERTBOT_BIN, @@ -1382,9 +1175,8 @@ def certbot_new( # Infomaniak and IONOS require RSA certificates with 4096-bit keys command.extend(["--rsa-key-size", "4096"]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using RSA-4096 for {provider} provider with " - f"{domains}") + debug_log(LOGGER, f"Using RSA-4096 for {provider} provider with " + f"{domains}") LOGGER.info(f"Using RSA-4096 for {provider} provider with {domains}") else: # Use elliptic curve certificates for all other providers @@ -1392,21 +1184,18 @@ def certbot_new( # Use P-384 elliptic curve for ZeroSSL certificates command.extend(["--elliptic-curve", "secp384r1"]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using ZeroSSL P-384 curve for {domains}") + debug_log(LOGGER, f"Using ZeroSSL P-384 curve for {domains}") LOGGER.info(f"Using ZeroSSL P-384 curve for {domains}") else: # Use P-256 elliptic curve for Let's Encrypt certificates command.extend(["--elliptic-curve", "secp256r1"]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using Let's Encrypt P-256 curve for {domains}") + debug_log(LOGGER, f"Using Let's Encrypt P-256 curve for {domains}") LOGGER.info(f"Using Let's Encrypt P-256 curve for {domains}") # Handle ZeroSSL EAB credentials if ca_provider.lower() == "zerossl": - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL detected as CA provider for {domains}") + debug_log(LOGGER, f"ZeroSSL detected as CA provider for {domains}") LOGGER.info(f"ZeroSSL detected as CA provider for {domains}") # Check for manually provided EAB credentials first @@ -1417,10 +1206,9 @@ def certbot_new( (getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") if server_name else "")) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Manual EAB credentials check:") - LOGGER.debug(f" EAB KID provided: {bool(eab_kid_env)}") - LOGGER.debug(f" EAB HMAC provided: {bool(eab_hmac_env)}") + debug_log(LOGGER, "Manual EAB credentials check:") + debug_log(LOGGER, f" EAB KID provided: {bool(eab_kid_env)}") + debug_log(LOGGER, f" EAB HMAC provided: {bool(eab_hmac_env)}") if eab_kid_env and eab_hmac_env: LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials " @@ -1430,9 +1218,8 @@ def certbot_new( LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid_env[:10]}...") elif api_key: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"ZeroSSL API key provided, setting up EAB " - f"credentials") + debug_log(LOGGER, f"ZeroSSL API key provided, setting up EAB " + f"credentials") LOGGER.info(f"ZeroSSL API key provided, setting up EAB " f"credentials") eab_kid, eab_hmac = setup_zerossl_eab_credentials(email, api_key) @@ -1461,11 +1248,10 @@ def certbot_new( if challenge_type == "dns": command.append("--preferred-challenges=dns") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"DNS challenge configuration:") - LOGGER.debug(f" Provider: {provider}") - LOGGER.debug(f" Propagation: {propagation}") - LOGGER.debug(f" Credentials path: {credentials_path}") + debug_log(LOGGER, "DNS challenge configuration:") + debug_log(LOGGER, f" Provider: {provider}") + debug_log(LOGGER, f" Propagation: {propagation}") + debug_log(LOGGER, f" Credentials path: {credentials_path}") if propagation != "default": if not propagation.isdigit(): @@ -1476,14 +1262,12 @@ def certbot_new( else: command.extend([f"--dns-{provider}-propagation-seconds", propagation]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Set DNS propagation time to " - f"{propagation} seconds") + debug_log(LOGGER, f"Set DNS propagation time to " + f"{propagation} seconds") if provider == "route53": - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Route53 provider - setting environment " - "variables") + debug_log(LOGGER, "Route53 provider - setting environment " + "variables") if credentials_path: with open(credentials_path, "r") as file: for line in file: @@ -1500,18 +1284,16 @@ def certbot_new( if provider in ("desec", "infomaniak", "ionos", "njalla", "scaleway"): command.extend(["--authenticator", f"dns-{provider}"]) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using explicit authenticator for {provider}") + debug_log(LOGGER, f"Using explicit authenticator for {provider}") else: command.append(f"--dns-{provider}") elif challenge_type == "http": - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - auth_hook = JOBS_PATH.joinpath('certbot-auth.py') - cleanup_hook = JOBS_PATH.joinpath('certbot-cleanup.py') - LOGGER.debug("HTTP challenge configuration:") - LOGGER.debug(f" Auth hook: {auth_hook}") - LOGGER.debug(f" Cleanup hook: {cleanup_hook}") + auth_hook = JOBS_PATH.joinpath('certbot-auth.py') + cleanup_hook = JOBS_PATH.joinpath('certbot-cleanup.py') + debug_log(LOGGER, "HTTP challenge configuration:") + debug_log(LOGGER, f" Auth hook: {auth_hook}") + debug_log(LOGGER, f" Cleanup hook: {cleanup_hook}") command.extend( [ @@ -1526,13 +1308,11 @@ def certbot_new( if force: command.append("--force-renewal") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Force renewal enabled") + debug_log(LOGGER, "Force renewal enabled") if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": command.append("-v") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Verbose mode enabled for certbot") + debug_log(LOGGER, "Verbose mode enabled for certbot") LOGGER.info(f"Executing certbot command for {domains}") # Show command but mask sensitive EAB values for security @@ -1548,19 +1328,17 @@ def certbot_new( else: safe_command.append(item) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Command: {' '.join(safe_command)}") - LOGGER.debug(f"Environment variables: {len(working_env)} items") - for key in working_env.keys(): - is_sensitive = any(sensitive in key.lower() - for sensitive in ['key', 'secret', 'token']) - value_display = '***MASKED***' if is_sensitive else 'set' - LOGGER.debug(f" {key}: {value_display}") + debug_log(LOGGER, f"Command: {' '.join(safe_command)}") + debug_log(LOGGER, f"Environment variables: {len(working_env)} items") + for key in working_env.keys(): + is_sensitive = any(sensitive in key.lower() + for sensitive in ['key', 'secret', 'token']) + value_display = '***MASKED***' if is_sensitive else 'set' + debug_log(LOGGER, f" {key}: {value_display}") LOGGER.info(f"Command: {' '.join(safe_command)}") current_date = datetime.now() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Starting certbot process") + debug_log(LOGGER, "Starting certbot process") process = Popen(command, stdin=DEVNULL, stderr=PIPE, universal_newlines=True, env=working_env) @@ -1586,15 +1364,13 @@ def certbot_new( ) current_date = datetime.now() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot still running, processed " - f"{lines_processed} output lines") + debug_log(LOGGER, f"Certbot still running, processed " + f"{lines_processed} output lines") final_return_code = process.returncode - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certbot process completed with return code: " - f"{final_return_code}") - LOGGER.debug(f"Total output lines processed: {lines_processed}") + debug_log(LOGGER, f"Certbot process completed with return code: " + f"{final_return_code}") + debug_log(LOGGER, f"Total output lines processed: {lines_processed}") return final_return_code @@ -1609,11 +1385,10 @@ def certbot_new( if isinstance(servers, str): servers = servers.split(" ") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Server configuration detected:") - LOGGER.debug(f" Multisite mode: {IS_MULTISITE}") - LOGGER.debug(f" Server count: {len(servers)}") - LOGGER.debug(f" Servers: {servers}") + debug_log(LOGGER, "Server configuration detected:") + debug_log(LOGGER, f" Multisite mode: {IS_MULTISITE}") + debug_log(LOGGER, f" Server count: {len(servers)}") + debug_log(LOGGER, f" Servers: {servers}") if not servers: LOGGER.warning("There are no server names, skipping generation...") @@ -1627,10 +1402,9 @@ def certbot_new( use_letsencrypt_dns = getenv("LETS_ENCRYPT_CHALLENGE", "http") == "dns" domains_server_names = {servers[0]: " ".join(servers).lower()} - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Single-site configuration:") - LOGGER.debug(f" Let's Encrypt enabled: {use_letsencrypt}") - LOGGER.debug(f" DNS challenge: {use_letsencrypt_dns}") + debug_log(LOGGER, "Single-site configuration:") + debug_log(LOGGER, f" Let's Encrypt enabled: {use_letsencrypt}") + debug_log(LOGGER, f" DNS challenge: {use_letsencrypt_dns}") else: domains_server_names = {} @@ -1650,13 +1424,12 @@ def certbot_new( server_name_env, first_server ).lower() - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Multi-site configuration:") - LOGGER.debug(f" Let's Encrypt enabled anywhere: " - f"{use_letsencrypt}") - LOGGER.debug(f" DNS challenge used anywhere: " - f"{use_letsencrypt_dns}") - LOGGER.debug(f" Domain mappings: {domains_server_names}") + debug_log(LOGGER, "Multi-site configuration:") + debug_log(LOGGER, f" Let's Encrypt enabled anywhere: " + f"{use_letsencrypt}") + debug_log(LOGGER, f" DNS challenge used anywhere: " + f"{use_letsencrypt_dns}") + debug_log(LOGGER, f" Domain mappings: {domains_server_names}") if not use_letsencrypt: LOGGER.info("Let's Encrypt is not activated, skipping generation...") @@ -1665,8 +1438,7 @@ def certbot_new( provider_classes = {} if use_letsencrypt_dns: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("DNS challenge detected - loading provider classes") + debug_log(LOGGER, "DNS challenge detected - loading provider classes") provider_classes: Dict[ str, @@ -1714,8 +1486,7 @@ def certbot_new( JOB = Job(LOGGER, __file__) # Restore data from db cache of certbot-renew job - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Restoring certificate data from database cache") + debug_log(LOGGER, "Restoring certificate data from database cache") JOB.restore_cache(job_name="certbot-renew") # Initialize environment variables for certbot execution @@ -1738,8 +1509,7 @@ def certbot_new( env_value: str = str(database_uri) env[env_key] = env_value - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking existing certificates") + debug_log(LOGGER, "Checking existing certificates") proc = run( [ @@ -1769,18 +1539,15 @@ def certbot_new( if proc.returncode != 0: LOGGER.error(f"Error while checking certificates:\n{proc.stdout}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate listing failed - proceeding anyway") + debug_log(LOGGER, "Certificate listing failed - proceeding anyway") else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate listing successful - analyzing " - "existing certificates") + debug_log(LOGGER, "Certificate listing successful - analyzing " + "existing certificates") certificate_blocks = stdout.split("Certificate Name: ")[1:] - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found {len(certificate_blocks)} existing " - f"certificates") + debug_log(LOGGER, f"Found {len(certificate_blocks)} existing " + f"certificates") for first_server, domains in domains_server_names.items(): auto_le_check = (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") @@ -1796,10 +1563,9 @@ def certbot_new( ) original_first_server = deepcopy(first_server) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing server: {first_server}") - LOGGER.debug(f" Challenge: {challenge_check}") - LOGGER.debug(f" Domains: {domains}") + debug_log(LOGGER, f"Processing server: {first_server}") + debug_log(LOGGER, f" Challenge: {challenge_check}") + debug_log(LOGGER, f" Domains: {domains}") wildcard_check = ( getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", @@ -1808,8 +1574,7 @@ def certbot_new( else getenv("USE_LETS_ENCRYPT_WILDCARD", "no") ) if (challenge_check == "dns" and wildcard_check == "yes"): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using wildcard mode for {first_server}") + debug_log(LOGGER, f"Using wildcard mode for {first_server}") wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( (first_server,) @@ -1867,11 +1632,10 @@ def certbot_new( else set(str(domains).split()) ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate domain comparison for " - f"{first_server}:") - LOGGER.debug(f" Existing: {sorted(str(d) for d in cert_domains_set)}") - LOGGER.debug(f" Desired: {sorted(str(d) for d in desired_domains_set)}") + debug_log(LOGGER, f"Certificate domain comparison for " + f"{first_server}:") + debug_log(LOGGER, f" Existing: {sorted(str(d) for d in cert_domains_set)}") + debug_log(LOGGER, f" Desired: {sorted(str(d) for d in desired_domains_set)}") if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 @@ -1911,10 +1675,9 @@ def certbot_new( ca_provider, staging_mode ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"CA server comparison for {first_server}:") - LOGGER.debug(f" Current: {current_server}") - LOGGER.debug(f" Expected: {expected_config['server']}") + debug_log(LOGGER, f"CA server comparison for {first_server}:") + debug_log(LOGGER, f" Current: {current_server}") + debug_log(LOGGER, f" Expected: {expected_config['server']}") if (current_server and current_server != expected_config["server"]): @@ -1934,10 +1697,9 @@ def certbot_new( use_staging = getenv(staging_env, "no") == "yes" is_test_cert = "TEST_CERT" in cert_domains.group("expiry_date") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Staging environment check for {first_server}:") - LOGGER.debug(f" Use staging: {use_staging}") - LOGGER.debug(f" Is test cert: {is_test_cert}") + debug_log(LOGGER, f"Staging environment check for {first_server}:") + debug_log(LOGGER, f" Use staging: {use_staging}") + debug_log(LOGGER, f" Is test cert: {is_test_cert}") staging_mismatch = ((is_test_cert and not use_staging) or (not is_test_cert and use_staging)) @@ -1973,10 +1735,9 @@ def certbot_new( current_provider = value.strip().replace("dns-", "") break - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Provider comparison for {first_server}:") - LOGGER.debug(f" Current: {current_provider}") - LOGGER.debug(f" Configured: {provider}") + debug_log(LOGGER, f"Provider comparison for {first_server}:") + debug_log(LOGGER, f" Current: {current_provider}") + debug_log(LOGGER, f" Configured: {provider}") if challenge_check == "dns": if provider and current_provider != provider: @@ -1990,9 +1751,8 @@ def certbot_new( # Check if DNS credentials have changed if provider and current_provider == provider: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Checking DNS credentials for " - f"{first_server}") + debug_log(LOGGER, f"Checking DNS credentials for " + f"{first_server}") credential_key = ( f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" @@ -2167,8 +1927,7 @@ def certbot_new( "credential_items": {}, } - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Service {first_server} configuration: {data}") + debug_log(LOGGER, f"Service {first_server} configuration: {data}") LOGGER.info(f"Service {first_server} configuration:") LOGGER.info(f" CA Provider: {data['ca_provider']}") @@ -2185,8 +1944,7 @@ def certbot_new( custom_profile = getenv(custom_profile_env, "").strip() if custom_profile: data["profile"] = custom_profile - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Using custom profile: {custom_profile}") + debug_log(LOGGER, f"Using custom profile: {custom_profile}") if data["challenge"] == "http" and data["use_wildcard"]: LOGGER.warning( @@ -2205,8 +1963,7 @@ def certbot_new( )) ) if should_skip_cert_check: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"No certificate needed for {first_server}") + debug_log(LOGGER, f"No certificate needed for {first_server}") continue if not data["max_retries"].isdigit(): @@ -2220,8 +1977,7 @@ def certbot_new( # Getting the DNS provider data if necessary if data["challenge"] == "dns": - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing DNS credentials for {first_server}") + debug_log(LOGGER, f"Processing DNS credentials for {first_server}") credential_key = ( f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" @@ -2300,15 +2056,14 @@ def certbot_new( ) data["credential_items"][key] = value - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - safe_data = data.copy() - masked_items = {k: "***MASKED***" - for k in data["credential_items"].keys()} - safe_data["credential_items"] = masked_items - if data["api_key"]: - safe_data["api_key"] = "***MASKED***" - LOGGER.debug(f"Safe data for service {first_server}: " - f"{dumps(safe_data)}") + safe_data = data.copy() + masked_items = {k: "***MASKED***" + for k in data["credential_items"].keys()} + safe_data["credential_items"] = masked_items + if data["api_key"]: + safe_data["api_key"] = "***MASKED***" + debug_log(LOGGER, f"Safe data for service {first_server}: " + f"{dumps(safe_data)}") # Validate CA provider and API key requirements api_key_status = 'Yes' if data['api_key'] else 'No' @@ -2402,13 +2157,11 @@ def certbot_new( if data["check_psl"]: if psl_lines is None: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Loading PSL for wildcard domain " - "validation") + debug_log(LOGGER, "Loading PSL for wildcard domain " + "validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Parsing PSL rules") + debug_log(LOGGER, "Parsing PSL rules") psl_rules = parse_psl(psl_lines) wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( @@ -2437,12 +2190,10 @@ def certbot_new( LOGGER.info(f"[{first_server}] Wildcard group {group}") elif data["check_psl"]: if psl_lines is None: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Loading PSL for regular domain validation") + debug_log(LOGGER, "Loading PSL for regular domain validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Parsing PSL rules") + debug_log(LOGGER, "Parsing PSL rules") psl_rules = parse_psl(psl_lines) for d in str(domains).split(): @@ -2455,17 +2206,15 @@ def certbot_new( break if is_blacklisted: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Skipping {first_server} due to PSL blacklist") + debug_log(LOGGER, f"Skipping {first_server} due to PSL blacklist") continue # Generating the credentials file credentials_path = CACHE_PATH.joinpath(*file_path) if data["challenge"] == "dns": - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Managing credentials file for {first_server}: " - f"{credentials_path}") + debug_log(LOGGER, f"Managing credentials file for {first_server}: " + f"{credentials_path}") if not credentials_path.is_file(): service_id = (first_server if not data["use_wildcard"] @@ -2523,9 +2272,8 @@ def certbot_new( credentials_path.chmod(0o600) if data["use_wildcard"]: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Wildcard processing complete for " - f"{first_server}") + debug_log(LOGGER, f"Wildcard processing complete for " + f"{first_server}") continue domains = str(domains).replace(" ", ",") @@ -2538,8 +2286,7 @@ def certbot_new( f"{data['profile']!r} profile..." ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Requesting certificate for {domains}") + debug_log(LOGGER, f"Requesting certificate for {domains}") cert_result = certbot_new_with_retry( data["challenge"], @@ -2574,8 +2321,7 @@ def certbot_new( # Generating the wildcards if necessary wildcards = WILDCARD_GENERATOR.get_wildcards() if wildcards: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing {len(wildcards)} wildcard groups") + debug_log(LOGGER, f"Processing {len(wildcards)} wildcard groups") for group, data in wildcards.items(): if not data: @@ -2587,11 +2333,10 @@ def certbot_new( profile = group_parts[2] base_domain = group_parts[3] - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Processing wildcard group: {group}") - LOGGER.debug(f" Provider: {provider}") - LOGGER.debug(f" Profile: {profile}") - LOGGER.debug(f" Base domain: {base_domain}") + debug_log(LOGGER, f"Processing wildcard group: {group}") + debug_log(LOGGER, f" Provider: {provider}") + debug_log(LOGGER, f" Profile: {profile}") + debug_log(LOGGER, f" Base domain: {base_domain}") email = data.pop("email") file_type = (provider_classes[provider].get_file_type() @@ -2642,9 +2387,8 @@ def certbot_new( base_domain = WILDCARD_GENERATOR.get_base_domain(domain) active_cert_names.add(base_domain) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Requesting wildcard certificate for " - f"{domains}") + debug_log(LOGGER, f"Requesting wildcard certificate for " + f"{domains}") wildcard_result = certbot_new_with_retry( "dns", @@ -2680,16 +2424,14 @@ def certbot_new( "generation..." ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate generation summary:") - LOGGER.debug(f" Generated: {certificates_generated}") - LOGGER.debug(f" Failed: {certificates_failed}") - LOGGER.debug(f" Total domains: {len(generated_domains)}") + debug_log(LOGGER, "Certificate generation summary:") + debug_log(LOGGER, f" Generated: {certificates_generated}") + debug_log(LOGGER, f" Failed: {certificates_failed}") + debug_log(LOGGER, f" Total domains: {len(generated_domains)}") if CACHE_PATH.is_dir(): # Clearing all missing credentials files - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Cleaning up old credentials files") + debug_log(LOGGER, "Cleaning up old credentials files") cleaned_files = 0 for ext in ("*.ini", "*.env", "*.json"): @@ -2707,8 +2449,7 @@ def certbot_new( ) cleaned_files += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Cleaned up {cleaned_files} old credentials files") + debug_log(LOGGER, f"Cleaned up {cleaned_files} old credentials files") # Clearing all no longer needed certificates if getenv("LETS_ENCRYPT_CLEAR_OLD_CERTS", "no") == "yes": @@ -2717,8 +2458,7 @@ def certbot_new( "used certificates..." ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Starting certificate cleanup process") + debug_log(LOGGER, "Starting certificate cleanup process") # Get list of all certificates proc = run( @@ -2744,11 +2484,10 @@ def certbot_new( certificate_blocks = proc.stdout.split("Certificate Name: ")[1:] certificates_removed = 0 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found {len(certificate_blocks)} certificates " - f"to evaluate") - LOGGER.debug(f"Active certificates: " - f"{sorted(str(name) for name in active_cert_names)}") + debug_log(LOGGER, f"Found {len(certificate_blocks)} certificates " + f"to evaluate") + debug_log(LOGGER, f"Active certificates: " + f"{sorted(str(name) for name in active_cert_names)}") for block in certificate_blocks: cert_name = block.split("\n", 1)[0].strip() @@ -2824,41 +2563,34 @@ def certbot_new( f"{delete_proc.stdout}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate cleanup completed - removed " - f"{certificates_removed} certificates") + debug_log(LOGGER, f"Certificate cleanup completed - removed " + f"{certificates_removed} certificates") else: LOGGER.error(f"Error listing certificates: {proc.stdout}") # Save data to db cache if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Saving certificate data to database cache") + debug_log(LOGGER, "Saving certificate data to database cache") cached, err = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") if not cached: LOGGER.error(f"Error while saving data to db cache: {err}") else: LOGGER.info("Successfully saved data to db cache") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Database cache update completed") + debug_log(LOGGER, "Database cache update completed") else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No certificate data to cache") + debug_log(LOGGER, "No certificate data to cache") except SystemExit as e: status = e.code - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Script exiting via SystemExit with code: {e.code}") + debug_log(LOGGER, f"Script exiting via SystemExit with code: {e.code}") except BaseException as e: status = 1 LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-new.py:\n{e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Script failed with unexpected exception") + debug_log(LOGGER, "Script failed with unexpected exception") -if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate generation process completed with status: " - f"{status}") +debug_log(LOGGER, f"Certificate generation process completed with status: " + f"{status}") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/certbot-renew.py b/src/common/core/letsencrypt/jobs/certbot-renew.py index ab92b633e0..d808a2640d 100644 --- a/src/common/core/letsencrypt/jobs/certbot-renew.py +++ b/src/common/core/letsencrypt/jobs/certbot-renew.py @@ -29,39 +29,43 @@ WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt") LOGS_DIR = join(sep, "var", "log", "bunkerweb", "letsencrypt") + +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + try: # Determine if Let's Encrypt is enabled in the current configuration # This checks both single-site and multi-site deployment modes - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Starting Let's Encrypt certificate renewal process") - LOGGER.debug("Checking if Let's Encrypt is enabled in configuration") - LOGGER.debug("Will check both single-site and multi-site modes") + debug_log(LOGGER, "Starting Let's Encrypt certificate renewal process") + debug_log(LOGGER, "Checking if Let's Encrypt is enabled in configuration") + debug_log(LOGGER, "Will check both single-site and multi-site modes") use_letsencrypt = False multisite_mode = getenv("MULTISITE", "no") == "yes" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Multisite mode detected: {multisite_mode}") - LOGGER.debug("Determining which Let's Encrypt check method to use") + debug_log(LOGGER, f"Multisite mode detected: {multisite_mode}") + debug_log(LOGGER, "Determining which Let's Encrypt check method to use") # Single-site mode: Check global AUTO_LETS_ENCRYPT setting if not multisite_mode: use_letsencrypt = getenv("AUTO_LETS_ENCRYPT", "no") == "yes" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking single-site mode configuration") - LOGGER.debug(f"Global AUTO_LETS_ENCRYPT setting: {use_letsencrypt}") - LOGGER.debug("Single setting controls all domains in this mode") + debug_log(LOGGER, "Checking single-site mode configuration") + debug_log(LOGGER, f"Global AUTO_LETS_ENCRYPT setting: {use_letsencrypt}") + debug_log(LOGGER, "Single setting controls all domains in this mode") # Multi-site mode: Check per-server AUTO_LETS_ENCRYPT settings else: server_names = getenv("SERVER_NAME", "www.example.com").split(" ") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking multi-site mode configuration") - LOGGER.debug(f"Found {len(server_names)} configured servers") - LOGGER.debug(f"Server list: {server_names}") - LOGGER.debug("Checking each server for Let's Encrypt enablement") + debug_log(LOGGER, "Checking multi-site mode configuration") + debug_log(LOGGER, f"Found {len(server_names)} configured servers") + debug_log(LOGGER, f"Server list: {server_names}") + debug_log(LOGGER, "Checking each server for Let's Encrypt enablement") # Check if any server has Let's Encrypt enabled for i, first_server in enumerate(server_names): @@ -69,44 +73,37 @@ server_le_enabled = getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") == "yes" - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug( - f"Server {i+1} ({first_server}): " - f"AUTO_LETS_ENCRYPT = {server_le_enabled}" - ) + debug_log(LOGGER, + f"Server {i+1} ({first_server}): " + f"AUTO_LETS_ENCRYPT = {server_le_enabled}") if server_le_enabled: use_letsencrypt = True - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Found Let's Encrypt enabled on {first_server}") - LOGGER.debug("At least one server needs renewal - proceeding") + debug_log(LOGGER, f"Found Let's Encrypt enabled on {first_server}") + debug_log(LOGGER, "At least one server needs renewal - proceeding") break # Exit early if Let's Encrypt is not configured if not use_letsencrypt: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Let's Encrypt not enabled on any servers") - LOGGER.debug("No certificates to renew - exiting early") - LOGGER.debug("Renewal process skipped entirely") + debug_log(LOGGER, "Let's Encrypt not enabled on any servers") + debug_log(LOGGER, "No certificates to renew - exiting early") + debug_log(LOGGER, "Renewal process skipped entirely") LOGGER.info("Let's Encrypt is not activated, skipping renew...") sys_exit(0) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Let's Encrypt is enabled - proceeding with renewal") - LOGGER.debug("Will attempt to renew all existing certificates") + debug_log(LOGGER, "Let's Encrypt is enabled - proceeding with renewal") + debug_log(LOGGER, "Will attempt to renew all existing certificates") # Initialize job handler for caching operations - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Initializing job handler for database operations") - LOGGER.debug("Job handler manages certificate data caching") + debug_log(LOGGER, "Initializing job handler for database operations") + debug_log(LOGGER, "Job handler manages certificate data caching") JOB = Job(LOGGER, __file__) # Set up environment variables for certbot execution # These control paths, timeouts, and configuration testing behavior - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Setting up environment for certbot execution") - LOGGER.debug("Configuring paths and operational parameters") + debug_log(LOGGER, "Setting up environment for certbot execution") + debug_log(LOGGER, "Configuring paths and operational parameters") env = { "PATH": getenv("PATH", ""), @@ -127,18 +124,17 @@ if database_uri is not None: env["DATABASE_URI"] = database_uri - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Environment configuration for certbot:") - path_display = (env['PATH'][:100] + "..." if len(env['PATH']) > 100 - else env['PATH']) - pythonpath_display = (env['PYTHONPATH'][:100] + "..." - if len(env['PYTHONPATH']) > 100 - else env['PYTHONPATH']) - LOGGER.debug(f" PATH: {path_display}") - LOGGER.debug(f" PYTHONPATH: {pythonpath_display}") - LOGGER.debug(f" RELOAD_MIN_TIMEOUT: {env['RELOAD_MIN_TIMEOUT']}") - LOGGER.debug(f" DISABLE_CONFIGURATION_TESTING: {env['DISABLE_CONFIGURATION_TESTING']}") - LOGGER.debug(f" DATABASE_URI configured: {'Yes' if database_uri else 'No'}") + debug_log(LOGGER, "Environment configuration for certbot:") + path_display = (env['PATH'][:100] + "..." if len(env['PATH']) > 100 + else env['PATH']) + pythonpath_display = (env['PYTHONPATH'][:100] + "..." + if len(env['PYTHONPATH']) > 100 + else env['PYTHONPATH']) + debug_log(LOGGER, f" PATH: {path_display}") + debug_log(LOGGER, f" PYTHONPATH: {pythonpath_display}") + debug_log(LOGGER, f" RELOAD_MIN_TIMEOUT: {env['RELOAD_MIN_TIMEOUT']}") + debug_log(LOGGER, f" DISABLE_CONFIGURATION_TESTING: {env['DISABLE_CONFIGURATION_TESTING']}") + debug_log(LOGGER, f" DATABASE_URI configured: {'Yes' if database_uri else 'No'}") # Construct certbot renew command with appropriate options # --no-random-sleep-on-renew: Prevents random delays in scheduled runs @@ -158,26 +154,23 @@ # Add verbose flag if debug logging is enabled if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": command.append("-v") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Debug mode enabled - adding verbose flag to certbot") - LOGGER.debug("Certbot will provide detailed output") + debug_log(LOGGER, "Debug mode enabled - adding verbose flag to certbot") + debug_log(LOGGER, "Certbot will provide detailed output") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certbot command configuration:") - LOGGER.debug(f" Command: {' '.join(command)}") - LOGGER.debug(f" Working directory: {WORK_DIR}") - LOGGER.debug(f" Config directory: {DATA_PATH.as_posix()}") - LOGGER.debug(f" Logs directory: {LOGS_DIR}") - LOGGER.debug("Command will check all existing certificates for renewal") + debug_log(LOGGER, "Certbot command configuration:") + debug_log(LOGGER, f" Command: {' '.join(command)}") + debug_log(LOGGER, f" Working directory: {WORK_DIR}") + debug_log(LOGGER, f" Config directory: {DATA_PATH.as_posix()}") + debug_log(LOGGER, f" Logs directory: {LOGS_DIR}") + debug_log(LOGGER, "Command will check all existing certificates for renewal") LOGGER.info("Starting certificate renewal process") # Execute certbot renew command # Process output is captured and logged through our logger - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Executing certbot renew command") - LOGGER.debug("Will capture and relay all certbot output") - LOGGER.debug("Process runs with isolated environment") + debug_log(LOGGER, "Executing certbot renew command") + debug_log(LOGGER, "Will capture and relay all certbot output") + debug_log(LOGGER, "Process runs with isolated environment") process = Popen( command, @@ -190,8 +183,7 @@ # Stream certbot output to our logger in real-time # This ensures all certbot messages are captured in BunkerWeb logs line_count = 0 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Starting real-time output capture from certbot") + debug_log(LOGGER, "Starting real-time output capture from certbot") while process.poll() is None: if process.stderr: @@ -199,99 +191,88 @@ line_count += 1 LOGGER_CERTBOT.info(line.strip()) - if (getenv("LOG_LEVEL", "INFO").upper() == "DEBUG" + if (getenv("LOG_LEVEL") == "debug" and line_count % 10 == 0): - LOGGER.debug(f"Processed {line_count} lines of certbot output") + debug_log(LOGGER, f"Processed {line_count} lines of certbot output") # Wait for process completion and check return code final_return_code = process.returncode - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certbot process completed") - LOGGER.debug(f"Final return code: {final_return_code}") - LOGGER.debug(f"Total output lines processed: {line_count}") - LOGGER.debug("Analyzing return code to determine success/failure") + debug_log(LOGGER, "Certbot process completed") + debug_log(LOGGER, f"Final return code: {final_return_code}") + debug_log(LOGGER, f"Total output lines processed: {line_count}") + debug_log(LOGGER, "Analyzing return code to determine success/failure") # Handle renewal results if final_return_code != 0: status = 2 LOGGER.error("Certificates renewal failed") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certbot returned non-zero exit code") - LOGGER.debug("Certificate renewal process failed") - LOGGER.debug("Will not cache certificate data due to failure") + debug_log(LOGGER, "Certbot returned non-zero exit code") + debug_log(LOGGER, "Certificate renewal process failed") + debug_log(LOGGER, "Will not cache certificate data due to failure") else: LOGGER.info("Certificate renewal completed successfully") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certbot completed successfully") - LOGGER.debug("All eligible certificates have been renewed") - LOGGER.debug("Proceeding to cache updated certificate data") + debug_log(LOGGER, "Certbot completed successfully") + debug_log(LOGGER, "All eligible certificates have been renewed") + debug_log(LOGGER, "Proceeding to cache updated certificate data") # Save Let's Encrypt certificate data to database cache # This ensures certificate data is available for distribution to cluster nodes - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Checking certificate data directory for caching") - LOGGER.debug(f"Certificate data path: {DATA_PATH}") - LOGGER.debug(f"Directory exists: {DATA_PATH.is_dir()}") - if DATA_PATH.is_dir(): - dir_contents = list(DATA_PATH.iterdir()) - LOGGER.debug(f"Directory contains {len(dir_contents)} items") - LOGGER.debug("Directory listing:") - for item in dir_contents[:5]: # Show first 5 items - LOGGER.debug(f" {item.name}") - if len(dir_contents) > 5: - LOGGER.debug(f" ... and {len(dir_contents) - 5} more items") + debug_log(LOGGER, "Checking certificate data directory for caching") + debug_log(LOGGER, f"Certificate data path: {DATA_PATH}") + debug_log(LOGGER, f"Directory exists: {DATA_PATH.is_dir()}") + if DATA_PATH.is_dir(): + dir_contents = list(DATA_PATH.iterdir()) + debug_log(LOGGER, f"Directory contains {len(dir_contents)} items") + debug_log(LOGGER, "Directory listing:") + for item in dir_contents[:5]: # Show first 5 items + debug_log(LOGGER, f" {item.name}") + if len(dir_contents) > 5: + debug_log(LOGGER, f" ... and {len(dir_contents) - 5} more items") # Only cache if directory exists and contains files if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate data found - proceeding with caching") - LOGGER.debug("This will store certificates in database for cluster distribution") + debug_log(LOGGER, "Certificate data found - proceeding with caching") + debug_log(LOGGER, "This will store certificates in database for cluster distribution") cached, err = JOB.cache_dir(DATA_PATH) if not cached: LOGGER.error( f"Error while saving Let's Encrypt data to db cache: {err}" ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Cache operation failed with error: {err}") - LOGGER.debug("Certificates renewed but not cached for distribution") + debug_log(LOGGER, f"Cache operation failed with error: {err}") + debug_log(LOGGER, "Certificates renewed but not cached for distribution") else: LOGGER.info("Successfully saved Let's Encrypt data to db cache") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Certificate data successfully cached to database") - LOGGER.debug("Cached certificates available for cluster distribution") + debug_log(LOGGER, "Certificate data successfully cached to database") + debug_log(LOGGER, "Cached certificates available for cluster distribution") else: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("No certificate data directory found or directory empty") - LOGGER.debug("This may be normal if no certificates needed renewal") + debug_log(LOGGER, "No certificate data directory found or directory empty") + debug_log(LOGGER, "This may be normal if no certificates needed renewal") LOGGER.warning("No certificate data found to cache") except SystemExit as e: status = e.code - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Script exiting via SystemExit with code: {e.code}") - LOGGER.debug("This is typically a normal exit condition") + debug_log(LOGGER, f"Script exiting via SystemExit with code: {e.code}") + debug_log(LOGGER, "This is typically a normal exit condition") except BaseException as e: status = 2 LOGGER.debug(format_exc()) LOGGER.error(f"Exception while running certbot-renew.py:\n{e}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug("Unexpected exception occurred during renewal") - LOGGER.debug("Full exception traceback logged above") - LOGGER.debug("Setting exit status to 2 due to unexpected exception") - LOGGER.debug("Renewal process aborted due to error") + debug_log(LOGGER, "Unexpected exception occurred during renewal") + debug_log(LOGGER, "Full exception traceback logged above") + debug_log(LOGGER, "Setting exit status to 2 due to unexpected exception") + debug_log(LOGGER, "Renewal process aborted due to error") -if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - LOGGER.debug(f"Certificate renewal process completed with final status: {status}") - if status == 0: - LOGGER.debug("Renewal process completed successfully") - LOGGER.debug("All certificates are up to date") - elif status == 2: - LOGGER.debug("Renewal process failed") - LOGGER.debug("Manual intervention may be required") - else: - LOGGER.debug(f"Renewal completed with status {status}") +debug_log(LOGGER, f"Certificate renewal process completed with final status: {status}") +if status == 0: + debug_log(LOGGER, "Renewal process completed successfully") + debug_log(LOGGER, "All certificates are up to date") +elif status == 2: + debug_log(LOGGER, "Renewal process failed") + debug_log(LOGGER, "Manual intervention may be required") +else: + debug_log(LOGGER, f"Renewal completed with status {status}") sys_exit(status) \ No newline at end of file diff --git a/src/common/core/letsencrypt/jobs/letsencrypt.py b/src/common/core/letsencrypt/jobs/letsencrypt.py index e50b0cd33e..3e66e10070 100644 --- a/src/common/core/letsencrypt/jobs/letsencrypt.py +++ b/src/common/core/letsencrypt/jobs/letsencrypt.py @@ -16,38 +16,34 @@ sys_path.append(python_path_str) +def debug_log(message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + print(f"[DEBUG] {message}") + + def alias_model_validator(field_map: dict): # Factory function for creating a model_validator for alias mapping. # This allows DNS providers to accept credentials under multiple field # names for better compatibility with different configuration formats. - # - # Args: - # field_map: Dictionary mapping canonical field names to list of - # aliases - # - # Returns: - # Configured model_validator function def validator(cls, values): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Processing aliases for {cls.__name__}") - print(f"DEBUG: Input values: {list(values.keys())}") - print(f"DEBUG: Field mapping has {len(field_map)} canonical fields") + debug_log(f"Processing aliases for {cls.__name__}") + debug_log(f"Input values: {list(values.keys())}") + debug_log(f"Field mapping has {len(field_map)} canonical fields") for field, aliases in field_map.items(): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Checking field '{field}' with {len(aliases)} aliases") + debug_log(f"Checking field '{field}' with {len(aliases)} aliases") for alias in aliases: if alias in values: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Found alias '{alias}' for field '{field}'") - print(f"DEBUG: Mapping alias '{alias}' to canonical field '{field}'") + debug_log(f"Found alias '{alias}' for field '{field}'") + debug_log(f"Mapping alias '{alias}' to canonical field '{field}'") values[field] = values[alias] break - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Final mapped values: {list(values.keys())}") - print(f"DEBUG: Alias processing completed for {cls.__name__}") + debug_log(f"Final mapped values: {list(values.keys())}") + debug_log(f"Alias processing completed for {cls.__name__}") return values @@ -64,24 +60,19 @@ class Provider(BaseModel): def get_formatted_credentials(self) -> bytes: # Return the formatted credentials to be written to a file. # Default implementation creates INI-style key=value format. - # - # Returns: - # bytes - UTF-8 encoded credential content - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - excluded_fields = {"file_type"} - fields = self.model_dump(exclude=excluded_fields) - print(f"DEBUG: {self.__class__.__name__} formatting {len(fields)} fields") - print(f"DEBUG: Excluded fields: {excluded_fields}") - print(f"DEBUG: Using default INI-style key=value format") + excluded_fields = {"file_type"} + fields = self.model_dump(exclude=excluded_fields) + debug_log(f"{self.__class__.__name__} formatting {len(fields)} fields") + debug_log(f"Excluded fields: {excluded_fields}") + debug_log("Using default INI-style key=value format") content = "\n".join( f"{key} = {value}" for key, value in self.model_dump(exclude={"file_type"}).items() ).encode("utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(content)} bytes of credential content") - print(f"DEBUG: Content will be written as UTF-8 encoded text") + debug_log(f"Generated {len(content)} bytes of credential content") + debug_log("Content will be written as UTF-8 encoded text") return content @@ -89,9 +80,6 @@ def get_formatted_credentials(self) -> bytes: def get_file_type() -> Literal["ini"]: # Return the file type that the credentials should be written to. # Default implementation returns 'ini' for most providers. - # - # Returns: - # Literal type indicating file extension return "ini" @@ -122,17 +110,13 @@ def get_formatted_credentials(self) -> bytes: # Return the formatted credentials, excluding defaults. # Only includes non-empty credential fields to avoid cluttering # output. - # - # Returns: - # bytes - UTF-8 encoded credential content - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - all_fields = self.model_dump(exclude={"file_type"}) - non_default_fields = self.model_dump( - exclude={"file_type"}, exclude_defaults=True - ) - print(f"DEBUG: Cloudflare provider has {len(all_fields)} total fields") - print(f"DEBUG: {len(non_default_fields)} non-default fields will be included") - print(f"DEBUG: Excluding empty/default values to minimize credential file") + all_fields = self.model_dump(exclude={"file_type"}) + non_default_fields = self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ) + debug_log(f"Cloudflare provider has {len(all_fields)} total fields") + debug_log(f"{len(non_default_fields)} non-default fields will be included") + debug_log("Excluding empty/default values to minimize credential file") content = "\n".join( f"{key} = {value}" @@ -141,8 +125,7 @@ def get_formatted_credentials(self) -> bytes: ).items() ).encode("utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(content)} bytes of Cloudflare credentials") + debug_log(f"Generated {len(content)} bytes of Cloudflare credentials") return content @@ -150,31 +133,25 @@ def get_formatted_credentials(self) -> bytes: def validate_cloudflare_credentials(self): # Validate Cloudflare credentials. # Ensures either API token or email+API key combination is provided. - # - # Raises: - # ValueError if neither authentication method is complete has_token = bool(self.dns_cloudflare_api_token) has_legacy = bool(self.dns_cloudflare_email and self.dns_cloudflare_api_key) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: Cloudflare credential validation:") - print(f"DEBUG: API token provided: {has_token}") - print(f"DEBUG: Legacy email+key provided: {has_legacy}") - print("DEBUG: At least one authentication method must be complete") + debug_log("Cloudflare credential validation:") + debug_log(f"API token provided: {has_token}") + debug_log(f"Legacy email+key provided: {has_legacy}") + debug_log("At least one authentication method must be complete") if not has_token and not has_legacy: - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: Neither authentication method is complete") - print("DEBUG: Validation will fail") + debug_log("Neither authentication method is complete") + debug_log("Validation will fail") raise ValueError( "Either 'dns_cloudflare_api_token' or both " "'dns_cloudflare_email' and 'dns_cloudflare_api_key' must be provided." ) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: Cloudflare credentials validation passed") - auth_method = "API token" if has_token else "email+API key" - print(f"DEBUG: Using {auth_method} authentication method") + debug_log("Cloudflare credentials validation passed") + auth_method = "API token" if has_token else "email+API key" + debug_log(f"Using {auth_method} authentication method") return self @@ -307,20 +284,15 @@ class GoogleProvider(Provider): def get_formatted_credentials(self) -> bytes: # Return the formatted credentials in JSON format. # Google Cloud requires credentials in JSON service account format. - # - # Returns: - # bytes - UTF-8 encoded JSON content - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: Google provider formatting credentials as JSON") - print("DEBUG: Using service account JSON format required by Google Cloud") + debug_log("Google provider formatting credentials as JSON") + debug_log("Using service account JSON format required by Google Cloud") json_content = self.model_dump_json( indent=2, exclude={"file_type"} ).encode("utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(json_content)} bytes of JSON credentials") - print("DEBUG: JSON format includes proper indentation for readability") + debug_log(f"Generated {len(json_content)} bytes of JSON credentials") + debug_log("JSON format includes proper indentation for readability") return json_content @@ -329,9 +301,6 @@ def get_file_type() -> Literal["json"]: # Return the file type that the credentials should be written to. # Google provider requires JSON format for service account # credentials. - # - # Returns: - # Literal type indicating JSON file extension return "json" @@ -464,17 +433,13 @@ class Rfc2136Provider(Provider): def get_formatted_credentials(self) -> bytes: # Return the formatted credentials, excluding defaults. # RFC2136 provider excludes default values to minimize configuration. - # - # Returns: - # bytes - UTF-8 encoded credential content - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - all_fields = self.model_dump(exclude={"file_type"}) - non_default_fields = self.model_dump( - exclude={"file_type"}, exclude_defaults=True - ) - print(f"DEBUG: RFC2136 provider has {len(all_fields)} total fields") - print(f"DEBUG: {len(non_default_fields)} non-default fields included") - print("DEBUG: Excluding defaults to minimize RFC2136 configuration") + all_fields = self.model_dump(exclude={"file_type"}) + non_default_fields = self.model_dump( + exclude={"file_type"}, exclude_defaults=True + ) + debug_log(f"RFC2136 provider has {len(all_fields)} total fields") + debug_log(f"{len(non_default_fields)} non-default fields included") + debug_log("Excluding defaults to minimize RFC2136 configuration") content = "\n".join( f"{key} = {value}" @@ -483,8 +448,7 @@ def get_formatted_credentials(self) -> bytes: ).items() ).encode("utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(content)} bytes of RFC2136 credentials") + debug_log(f"Generated {len(content)} bytes of RFC2136 credentials") return content @@ -511,22 +475,17 @@ class Route53Provider(Provider): def get_formatted_credentials(self) -> bytes: # Return the formatted credentials in environment variable format. # Route53 uses environment variables for AWS credentials. - # - # Returns: - # bytes - UTF-8 encoded environment variable format - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - fields = self.model_dump(exclude={"file_type"}) - print(f"DEBUG: Route53 provider formatting {len(fields)} fields as env vars") - print("DEBUG: Using environment variable format for AWS credentials") + fields = self.model_dump(exclude={"file_type"}) + debug_log(f"Route53 provider formatting {len(fields)} fields as env vars") + debug_log("Using environment variable format for AWS credentials") content = "\n".join( f"{key.upper()}={value!r}" for key, value in self.model_dump(exclude={"file_type"}).items() ).encode("utf-8") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(content)} bytes of environment variables") - print("DEBUG: Variables will be uppercase as per AWS convention") + debug_log(f"Generated {len(content)} bytes of environment variables") + debug_log("Variables will be uppercase as per AWS convention") return content @@ -534,9 +493,6 @@ def get_formatted_credentials(self) -> bytes: def get_file_type() -> Literal["env"]: # Return the file type that the credentials should be written to. # Route53 provider uses environment variable format. - # - # Returns: - # Literal type indicating env file extension return "env" @@ -596,34 +552,25 @@ class WildcardGenerator: # for efficient certificate management across multiple subdomains. def __init__(self): - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: Initializing WildcardGenerator") - print("DEBUG: Setting up empty domain groups and wildcard storage") + debug_log("Initializing WildcardGenerator") + debug_log("Setting up empty domain groups and wildcard storage") # Stores raw domains grouped by identifier self.__domain_groups = {} # Stores generated wildcard patterns self.__wildcards = {} - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print("DEBUG: WildcardGenerator initialized with empty groups") - print("DEBUG: Ready to accept domain groups for wildcard generation") + debug_log("WildcardGenerator initialized with empty groups") + debug_log("Ready to accept domain groups for wildcard generation") def extend(self, group: str, domains: List[str], email: str, staging: bool = False): # Add domains to a group and regenerate wildcards. # Organizes domains by group and environment for wildcard generation. - # - # Args: - # group: Group identifier for these domains - # domains: List of domains to add - # email: Contact email for this domain group - # staging: Whether these domains are for staging environment - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Extending group '{group}' with {len(domains)} domains") - print(f"DEBUG: Environment: {'staging' if staging else 'production'}") - print(f"DEBUG: Contact email: {email}") - print(f"DEBUG: Domain list: {domains}") + debug_log(f"Extending group '{group}' with {len(domains)} domains") + debug_log(f"Environment: {'staging' if staging else 'production'}") + debug_log(f"Contact email: {email}") + debug_log(f"Domain list: {domains}") # Initialize group if it doesn't exist if group not in self.__domain_groups: @@ -632,9 +579,8 @@ def extend(self, group: str, domains: List[str], email: str, "prod": set(), "email": email } - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Created new domain group '{group}'") - print("DEBUG: Group initialized with empty staging and prod sets") + debug_log(f"Created new domain group '{group}'") + debug_log("Group initialized with empty staging and prod sets") # Add domains to appropriate environment env_type = "staging" if staging else "prod" @@ -644,11 +590,10 @@ def extend(self, group: str, domains: List[str], email: str, self.__domain_groups[group][env_type].add(domain) domains_added += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added {domains_added} valid domains to {env_type} environment") - total_staging = len(self.__domain_groups[group]["staging"]) - total_prod = len(self.__domain_groups[group]["prod"]) - print(f"DEBUG: Group '{group}' totals: {total_staging} staging, {total_prod} prod domains") + debug_log(f"Added {domains_added} valid domains to {env_type} environment") + total_staging = len(self.__domain_groups[group]["staging"]) + total_prod = len(self.__domain_groups[group]["prod"]) + debug_log(f"Group '{group}' totals: {total_staging} staging, {total_prod} prod domains") # Regenerate wildcards after adding new domains self.__generate_wildcards(staging) @@ -657,17 +602,12 @@ def __generate_wildcards(self, staging: bool = False): # Generate wildcard patterns for the specified environment. # Creates optimized wildcard certificates that cover multiple # subdomains. - # - # Args: - # staging: Whether to generate wildcards for staging environment - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - env_type = "staging" if staging else "prod" - print(f"DEBUG: Generating wildcards for {env_type} environment") - print(f"DEBUG: Processing {len(self.__domain_groups)} domain groups") - print("DEBUG: Will convert subdomains to wildcard patterns") + env_type = "staging" if staging else "prod" + debug_log(f"Generating wildcards for {env_type} environment") + debug_log(f"Processing {len(self.__domain_groups)} domain groups") + debug_log("Will convert subdomains to wildcard patterns") self.__wildcards.clear() - env_type = "staging" if staging else "prod" wildcards_generated = 0 # Process each domain group @@ -685,27 +625,19 @@ def __generate_wildcards(self, staging: bool = False): self.__add_domain_wildcards(domain, group, env_type) wildcards_generated += 1 - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated wildcard patterns for {wildcards_generated} domains") - print("DEBUG: Wildcard generation completed") + debug_log(f"Generated wildcard patterns for {wildcards_generated} domains") + debug_log("Wildcard generation completed") def __add_domain_wildcards(self, domain: str, group: str, env_type: str): # Convert a domain to wildcard patterns and add to the wildcards # collection. Determines optimal wildcard patterns based on domain # structure. - # - # Args: - # domain: Domain to process - # group: Group identifier - # env_type: Environment type (staging or prod) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Processing domain '{domain}' for wildcard patterns") + debug_log(f"Processing domain '{domain}' for wildcard patterns") parts = domain.split(".") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Domain has {len(parts)} parts: {parts}") - print("DEBUG: Analyzing domain structure for wildcard generation") + debug_log(f"Domain has {len(parts)} parts: {parts}") + debug_log("Analyzing domain structure for wildcard generation") # Handle subdomains (domains with more than 2 parts) if len(parts) > 2: @@ -716,28 +648,22 @@ def __add_domain_wildcards(self, domain: str, group: str, env_type: str): self.__wildcards[group][env_type].add(wildcard_domain) self.__wildcards[group][env_type].add(base_domain) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Subdomain detected - created wildcard '{wildcard_domain}'") - print(f"DEBUG: Also added base domain '{base_domain}'") - print("DEBUG: Wildcard will cover all subdomains of base") + debug_log(f"Subdomain detected - created wildcard '{wildcard_domain}'") + debug_log(f"Also added base domain '{base_domain}'") + debug_log("Wildcard will cover all subdomains of base") else: # Just add the raw domain for top-level domains self.__wildcards[group][env_type].add(domain) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Top-level domain - added '{domain}' directly") - print("DEBUG: No wildcard needed for top-level domain") + debug_log(f"Top-level domain - added '{domain}' directly") + debug_log("No wildcard needed for top-level domain") def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", "email"], str]]: # Get formatted wildcard domains for each group. # Returns organized wildcard data ready for certificate generation. - # - # Returns: - # Dictionary of group data with formatted wildcard domains - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Formatting wildcards for {len(self.__wildcards)} groups") - print("DEBUG: Converting wildcard sets to comma-separated strings") + debug_log(f"Formatting wildcards for {len(self.__wildcards)} groups") + debug_log("Converting wildcard sets to comma-separated strings") result = {} total_domains = 0 @@ -752,13 +678,11 @@ def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", result[group][env_type] = ",".join(sorted_domains) total_domains += len(domains) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Group '{group}' {env_type}: {len(domains)} domains") - print(f"DEBUG: Sorted with wildcards first: {sorted_domains[:3]}...") + debug_log(f"Group '{group}' {env_type}: {len(domains)} domains") + debug_log(f"Sorted with wildcards first: {sorted_domains[:3]}...") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Formatted {total_domains} total wildcard domains") - print("DEBUG: Ready for certificate generation") + debug_log(f"Formatted {total_domains} total wildcard domains") + debug_log("Ready for certificate generation") return result @@ -766,44 +690,33 @@ def get_wildcards(self) -> Dict[str, Dict[Literal["staging", "prod", def extract_wildcards_from_domains(domains: List[str]) -> List[str]: # Generate wildcard patterns from a list of domains. # Static method for generating wildcards without managing groups. - # - # Args: - # domains: List of domains to process - # - # Returns: - # List of extracted wildcard domains - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Extracting wildcards from {len(domains)} domains") - print(f"DEBUG: Input domains: {domains}") - print("DEBUG: Static method - no group management") + debug_log(f"Extracting wildcards from {len(domains)} domains") + debug_log(f"Input domains: {domains}") + debug_log("Static method - no group management") wildcards = set() for domain in domains: parts = domain.split(".") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Processing '{domain}' with {len(parts)} parts") + debug_log(f"Processing '{domain}' with {len(parts)} parts") # Generate wildcards for subdomains if len(parts) > 2: base_domain = ".".join(parts[1:]) wildcards.add(f"*.{base_domain}") wildcards.add(base_domain) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added wildcard *.{base_domain} and base {base_domain}") + debug_log(f"Added wildcard *.{base_domain} and base {base_domain}") else: # Just add the domain for top-level domains wildcards.add(domain) - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Added top-level domain {domain} directly") + debug_log(f"Added top-level domain {domain} directly") # Sort with wildcards first result = sorted(wildcards, key=lambda x: x[0] != "*") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Generated {len(result)} wildcard patterns") - print(f"DEBUG: Final result: {result}") + debug_log(f"Generated {len(result)} wildcard patterns") + debug_log(f"Final result: {result}") return result @@ -811,19 +724,12 @@ def extract_wildcards_from_domains(domains: List[str]) -> List[str]: def get_base_domain(domain: str) -> str: # Extract the base domain from a domain name. # Removes wildcard prefix if present to get the actual domain. - # - # Args: - # domain: Input domain name - # - # Returns: - # Base domain (without wildcard prefix if present) base = domain.lstrip("*.") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - if domain != base: - print(f"DEBUG: Extracted base domain '{base}' from wildcard '{domain}'") - else: - print(f"DEBUG: Domain '{domain}' is already a base domain") + if domain != base: + debug_log(f"Extracted base domain '{base}' from wildcard '{domain}'") + else: + debug_log(f"Domain '{domain}' is already a base domain") return base @@ -834,23 +740,11 @@ def create_group_name(domain: str, provider: str, challenge_type: str, # Generate a consistent group name for wildcards. # Creates a unique identifier for grouping related wildcard # certificates. - # - # Args: - # domain: The domain name - # provider: DNS provider name or 'http' for HTTP challenge - # challenge_type: Challenge type (dns or http) - # staging: Whether this is for staging environment - # content_hash: Hash of credential content - # profile: Certificate profile (classic, tlsserver or shortlived) - # - # Returns: - # A formatted group name string - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Creating group name for domain '{domain}'") - print(f"DEBUG: Provider: {provider}, Challenge: {challenge_type}") - print(f"DEBUG: Environment: {'staging' if staging else 'production'}") - print(f"DEBUG: Profile: {profile}") - print(f"DEBUG: Content hash: {content_hash[:10]}... (truncated)") + debug_log(f"Creating group name for domain '{domain}'") + debug_log(f"Provider: {provider}, Challenge: {challenge_type}") + debug_log(f"Environment: {'staging' if staging else 'production'}") + debug_log(f"Profile: {profile}") + debug_log(f"Content hash: {content_hash[:10]}... (truncated)") # Extract base domain and format it for the group name base_domain = WildcardGenerator.get_base_domain(domain).replace(".", @@ -863,10 +757,9 @@ def create_group_name(domain: str, provider: str, challenge_type: str, group_name = (f"{challenge_identifier}_{env}_{profile}_{base_domain}_" f"{content_hash}") - if getenv("LOG_LEVEL", "INFO").upper() == "DEBUG": - print(f"DEBUG: Base domain formatted: {base_domain}") - print(f"DEBUG: Challenge identifier: {challenge_identifier}") - print(f"DEBUG: Generated group name: '{group_name}'") - print("DEBUG: Group name ensures consistent certificate grouping") + debug_log(f"Base domain formatted: {base_domain}") + debug_log(f"Challenge identifier: {challenge_identifier}") + debug_log(f"Generated group name: '{group_name}'") + debug_log("Group name ensures consistent certificate grouping") return group_name \ No newline at end of file diff --git a/src/common/core/letsencrypt/letsencrypt.lua b/src/common/core/letsencrypt/letsencrypt.lua index 09646547da..d7bb9ceb88 100644 --- a/src/common/core/letsencrypt/letsencrypt.lua +++ b/src/common/core/letsencrypt/letsencrypt.lua @@ -28,259 +28,397 @@ local decode = cjson.decode local execute = os.execute local remove = os.remove +-- Log debug messages only when LOG_LEVEL environment variable is set to +-- "debug" +local function debug_log(logger, message) + if os.getenv("LOG_LEVEL") == "debug" then + logger:log(NOTICE, "[DEBUG] " .. message) + end +end + +-- Initialize the letsencrypt plugin with the given context function letsencrypt:initialize(ctx) - -- Call parent initialize - plugin.initialize(self, "letsencrypt", ctx) + # Call parent initialize + plugin.initialize(self, "letsencrypt", ctx) end +-- Set the https_configured flag based on AUTO_LETS_ENCRYPT variable function letsencrypt:set() - local https_configured = self.variables["AUTO_LETS_ENCRYPT"] - if https_configured == "yes" then - self.ctx.bw.https_configured = "yes" - end - return self:ret(true, "set https_configured to " .. https_configured) + local https_configured = self.variables["AUTO_LETS_ENCRYPT"] + if https_configured == "yes" then + self.ctx.bw.https_configured = "yes" + end + debug_log(self.logger, "Set https_configured to " .. https_configured) + return self:ret(true, "set https_configured to " .. https_configured) end +-- Initialize SSL certificates and load them into the datastore function letsencrypt:init() - local ret_ok, ret_err = true, "success" - local wildcard_servers = {} + local ret_ok, ret_err = true, "success" + local wildcard_servers = {} + + debug_log(self.logger, "Starting letsencrypt init phase") - if has_variable("AUTO_LETS_ENCRYPT", "yes") then - local multisite, err = get_variable("MULTISITE", false) - if not multisite then - return self:ret(false, "can't get MULTISITE variable : " .. err) - end - if multisite == "yes" then - local vars - vars, err = get_multiple_variables({ - "AUTO_LETS_ENCRYPT", - "LETS_ENCRYPT_CHALLENGE", - "LETS_ENCRYPT_DNS_PROVIDER", - "USE_LETS_ENCRYPT_WILDCARD", - "SERVER_NAME", - }) - if not vars then - return self:ret(false, "can't get required variables : " .. err) - end - local credential_items - credential_items, err = get_multiple_variables({ "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" }) - if not credential_items then - return self:ret(false, "can't get credential items : " .. err) - end - for server_name, multisite_vars in pairs(vars) do - if - multisite_vars["AUTO_LETS_ENCRYPT"] == "yes" - and server_name ~= "global" - and ( - multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "http" - or ( - multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "dns" - and multisite_vars["LETS_ENCRYPT_DNS_PROVIDER"] ~= "" - and credential_items[server_name] - ) - ) - then - local data - if - multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "dns" - and multisite_vars["USE_LETS_ENCRYPT_WILDCARD"] == "yes" - then - for part in server_name:gmatch("%S+") do - wildcard_servers[part] = true - end - local parts = {} - for part in server_name:gmatch("[^.]+") do - table.insert(parts, part) - end - server_name = table.concat(parts, ".", 2) - data = self.datastore:get("plugin_letsencrypt_" .. server_name, true) - else - for part in server_name:gmatch("%S+") do - wildcard_servers[part] = false - end - end - if not data then - -- Load certificate - local check - check, data = read_files({ - "/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/fullchain.pem", - "/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/privkey.pem", - }) - if not check then - self.logger:log(ERR, "error while reading files : " .. data) - ret_ok = false - ret_err = "error reading files" - else - if - multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "dns" - and multisite_vars["USE_LETS_ENCRYPT_WILDCARD"] == "yes" - then - check, err = self:load_data(data, server_name) - else - check, err = self:load_data(data, multisite_vars["SERVER_NAME"]) - end - if not check then - self.logger:log(ERR, "error while loading data : " .. err) - ret_ok = false - ret_err = "error loading data" - end - end - end - end - end - else - local server_name - server_name, err = get_variable("SERVER_NAME", false) - if not server_name then - return self:ret(false, "can't get SERVER_NAME variable : " .. err) - end - local use_wildcard - use_wildcard, err = get_variable("USE_LETS_ENCRYPT_WILDCARD", false) - if not use_wildcard then - return self:ret(false, "can't get USE_LETS_ENCRYPT_WILDCARD variable : " .. err) - end - local challenge - challenge, err = get_variable("LETS_ENCRYPT_CHALLENGE", false) - if not challenge then - return self:ret(false, "can't get LETS_ENCRYPT_CHALLENGE variable : " .. err) - end - server_name = server_name:match("%S+") - if challenge == "dns" and use_wildcard == "yes" then - for part in server_name:gmatch("%S+") do - wildcard_servers[part] = true - end - local parts = {} - for part in server_name:gmatch("[^.]+") do - table.insert(parts, part) - end - server_name = table.concat(parts, ".", 2) - else - for part in server_name:gmatch("%S+") do - wildcard_servers[part] = false - end - end - local check, data = read_files({ - "/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/fullchain.pem", - "/var/cache/bunkerweb/letsencrypt/etc/live/" .. server_name .. "/privkey.pem", - }) - if not check then - self.logger:log(ERR, "error while reading files : " .. data) - ret_ok = false - ret_err = "error reading files" - else - check, err = self:load_data(data, server_name) - if not check then - self.logger:log(ERR, "error while loading data : " .. err) - ret_ok = false - ret_err = "error loading data" - end - end - end - else - ret_err = "let's encrypt is not used" - end + if has_variable("AUTO_LETS_ENCRYPT", "yes") then + debug_log(self.logger, "AUTO_LETS_ENCRYPT is enabled") + + local multisite, err = get_variable("MULTISITE", false) + if not multisite then + return self:ret(false, "can't get MULTISITE variable : " .. err) + end + + debug_log(self.logger, "MULTISITE mode is " .. multisite) + + if multisite == "yes" then + debug_log(self.logger, "Processing multisite configuration") + + local vars + vars, err = get_multiple_variables({ + "AUTO_LETS_ENCRYPT", + "LETS_ENCRYPT_CHALLENGE", + "LETS_ENCRYPT_DNS_PROVIDER", + "USE_LETS_ENCRYPT_WILDCARD", + "SERVER_NAME", + }) + if not vars then + return self:ret(false, + "can't get required variables : " .. err) + end + + local credential_items + credential_items, err = get_multiple_variables({ + "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" + }) + if not credential_items then + return self:ret(false, + "can't get credential items : " .. err) + end + + for server_name, multisite_vars in pairs(vars) do + debug_log(self.logger, + "Processing server: " .. server_name) + + if multisite_vars["AUTO_LETS_ENCRYPT"] == "yes" + and server_name ~= "global" + and ( + multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "http" + or ( + multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "dns" + and multisite_vars["LETS_ENCRYPT_DNS_PROVIDER"] + ~= "" + and credential_items[server_name] + ) + ) + then + debug_log(self.logger, + "Server " .. server_name .. " qualifies for SSL") + + local data + if multisite_vars["LETS_ENCRYPT_CHALLENGE"] == "dns" + and multisite_vars["USE_LETS_ENCRYPT_WILDCARD"] + == "yes" + then + debug_log(self.logger, + "Using wildcard configuration for " .. server_name) + + for part in server_name:gmatch("%S+") do + wildcard_servers[part] = true + end + local parts = {} + for part in server_name:gmatch("[^.]+") do + table.insert(parts, part) + end + server_name = table.concat(parts, ".", 2) + data = self.datastore:get("plugin_letsencrypt_" .. + server_name, true) + else + for part in server_name:gmatch("%S+") do + wildcard_servers[part] = false + end + end + + if not data then + debug_log(self.logger, + "Loading certificate files for " .. server_name) + + # Load certificate + local check + check, data = read_files({ + "/var/cache/bunkerweb/letsencrypt/etc/live/" .. + server_name .. "/fullchain.pem", + "/var/cache/bunkerweb/letsencrypt/etc/live/" .. + server_name .. "/privkey.pem", + }) + if not check then + self.logger:log(ERR, + "error while reading files : " .. data) + ret_ok = false + ret_err = "error reading files" + else + if multisite_vars["LETS_ENCRYPT_CHALLENGE"] + == "dns" + and multisite_vars["USE_LETS_ENCRYPT_WILDCARD"] + == "yes" + then + check, err = self:load_data(data, server_name) + else + check, err = self:load_data(data, + multisite_vars["SERVER_NAME"]) + end + if not check then + self.logger:log(ERR, + "error while loading data : " .. err) + ret_ok = false + ret_err = "error loading data" + end + end + end + end + end + else + debug_log(self.logger, "Processing single-site configuration") + + local server_name + server_name, err = get_variable("SERVER_NAME", false) + if not server_name then + return self:ret(false, + "can't get SERVER_NAME variable : " .. err) + end + + local use_wildcard + use_wildcard, err = get_variable("USE_LETS_ENCRYPT_WILDCARD", + false) + if not use_wildcard then + return self:ret(false, + "can't get USE_LETS_ENCRYPT_WILDCARD variable : " .. err) + end + + local challenge + challenge, err = get_variable("LETS_ENCRYPT_CHALLENGE", false) + if not challenge then + return self:ret(false, + "can't get LETS_ENCRYPT_CHALLENGE variable : " .. err) + end + + server_name = server_name:match("%S+") + debug_log(self.logger, + "Processing server_name: " .. server_name) + + if challenge == "dns" and use_wildcard == "yes" then + debug_log(self.logger, + "Using wildcard DNS challenge for " .. server_name) + + for part in server_name:gmatch("%S+") do + wildcard_servers[part] = true + end + local parts = {} + for part in server_name:gmatch("[^.]+") do + table.insert(parts, part) + end + server_name = table.concat(parts, ".", 2) + else + for part in server_name:gmatch("%S+") do + wildcard_servers[part] = false + end + end + + debug_log(self.logger, + "Loading certificates for " .. server_name) + + local check, data = read_files({ + "/var/cache/bunkerweb/letsencrypt/etc/live/" .. + server_name .. "/fullchain.pem", + "/var/cache/bunkerweb/letsencrypt/etc/live/" .. + server_name .. "/privkey.pem", + }) + if not check then + self.logger:log(ERR, "error while reading files : " .. data) + ret_ok = false + ret_err = "error reading files" + else + check, err = self:load_data(data, server_name) + if not check then + self.logger:log(ERR, "error while loading data : " .. err) + ret_ok = false + ret_err = "error loading data" + end + end + end + else + debug_log(self.logger, "Let's Encrypt is not enabled") + ret_err = "let's encrypt is not used" + end - local ok, err = self.datastore:set("plugin_letsencrypt_wildcard_servers", wildcard_servers, nil, true) - if not ok then - return self:ret(false, "error while setting wildcard servers into datastore : " .. err) - end + debug_log(self.logger, "Storing wildcard servers configuration") + + local ok, err = self.datastore:set("plugin_letsencrypt_wildcard_servers", + wildcard_servers, nil, true) + if not ok then + return self:ret(false, + "error while setting wildcard servers into datastore : " .. err) + end - return self:ret(ret_ok, ret_err) + debug_log(self.logger, "Init phase completed with status: " .. + tostring(ret_ok)) + + return self:ret(ret_ok, ret_err) end +-- Handle SSL certificate selection based on SNI function letsencrypt:ssl_certificate() - local server_name, err = ssl_server_name() - if not server_name then - return self:ret(false, "can't get server_name : " .. err) - end - local wildcard_servers, err = self.datastore:get("plugin_letsencrypt_wildcard_servers", true) - if not wildcard_servers then - return self:ret(false, "can't get wildcard servers : " .. err) - end - if wildcard_servers[server_name] then - local parts = {} - for part in server_name:gmatch("[^.]+") do - table.insert(parts, part) - end - server_name = table.concat(parts, ".", 2) - end - local data - data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true) - if not data and err ~= "not found" then - return self:ret(false, "error while getting plugin_letsencrypt_" .. server_name .. " from datastore : " .. err) - elseif data then - return self:ret(true, "certificate/key data found", data) - end - return self:ret(true, "let's encrypt is not used") + debug_log(self.logger, "SSL certificate phase started") + + local server_name, err = ssl_server_name() + if not server_name then + return self:ret(false, "can't get server_name : " .. err) + end + + debug_log(self.logger, "Processing SSL for server: " .. server_name) + + local wildcard_servers, err = self.datastore:get( + "plugin_letsencrypt_wildcard_servers", true) + if not wildcard_servers then + return self:ret(false, "can't get wildcard servers : " .. err) + end + + if wildcard_servers[server_name] then + debug_log(self.logger, + "Using wildcard certificate for " .. server_name) + + local parts = {} + for part in server_name:gmatch("[^.]+") do + table.insert(parts, part) + end + server_name = table.concat(parts, ".", 2) + end + + local data + data, err = self.datastore:get("plugin_letsencrypt_" .. server_name, true) + if not data and err ~= "not found" then + return self:ret(false, "error while getting plugin_letsencrypt_" .. + server_name .. " from datastore : " .. err) + elseif data then + debug_log(self.logger, + "Certificate data found for " .. server_name) + return self:ret(true, "certificate/key data found", data) + end + + debug_log(self.logger, "No certificate data found for " .. server_name) + return self:ret(true, "let's encrypt is not used") end +-- Load certificate and private key data into the datastore function letsencrypt:load_data(data, server_name) - -- Load certificate - local cert_chain, err = parse_pem_cert(data[1]) - if not cert_chain then - return false, "error while parsing pem cert : " .. err - end - -- Load key - local priv_key - priv_key, err = parse_pem_priv_key(data[2]) - if not priv_key then - return false, "error while parsing pem priv key : " .. err - end - -- Cache data - for key in server_name:gmatch("%S+") do - local cache_key = "plugin_letsencrypt_" .. key - local ok - ok, err = self.datastore:set(cache_key, { cert_chain, priv_key }, nil, true) - if not ok then - return false, "error while setting data into datastore : " .. err - end - end - return true + debug_log(self.logger, "Loading certificate data for " .. server_name) + + # Load certificate + local cert_chain, err = parse_pem_cert(data[1]) + if not cert_chain then + return false, "error while parsing pem cert : " .. err + end + + # Load key + local priv_key + priv_key, err = parse_pem_priv_key(data[2]) + if not priv_key then + return false, "error while parsing pem priv key : " .. err + end + + debug_log(self.logger, "Certificate and key parsed successfully") + + # Cache data + for key in server_name:gmatch("%S+") do + debug_log(self.logger, "Caching certificate data for " .. key) + + local cache_key = "plugin_letsencrypt_" .. key + local ok + ok, err = self.datastore:set(cache_key, { cert_chain, priv_key }, + nil, true) + if not ok then + return false, "error while setting data into datastore : " .. err + end + end + + debug_log(self.logger, "Certificate data cached successfully") + return true end +-- Handle ACME challenge requests during certificate generation function letsencrypt:access() - if - self.variables["LETS_ENCRYPT_PASSTHROUGH"] == "no" - and sub(self.ctx.bw.uri, 1, string.len("/.well-known/acme-challenge/")) == "/.well-known/acme-challenge/" - then - self.logger:log(NOTICE, "got a visit from Let's Encrypt, let's whitelist it") - return self:ret(true, "visit from LE", OK) - end - return self:ret(true, "success") + debug_log(self.logger, "Access phase started") + + if self.variables["LETS_ENCRYPT_PASSTHROUGH"] == "no" + and sub(self.ctx.bw.uri, 1, + string.len("/.well-known/acme-challenge/")) == + "/.well-known/acme-challenge/" + then + debug_log(self.logger, "ACME challenge request detected") + + self.logger:log(NOTICE, + "got a visit from Let's Encrypt, let's whitelist it") + return self:ret(true, "visit from LE", OK) + end + + return self:ret(true, "success") end --- luacheck: ignore 212 +-- Handle API requests for certificate challenge management function letsencrypt:api() - if - not match(self.ctx.bw.uri, "^/lets%-encrypt/challenge$") - or (self.ctx.bw.request_method ~= "POST" and self.ctx.bw.request_method ~= "DELETE") - then - return self:ret(false, "success") - end - local acme_folder = "/var/tmp/bunkerweb/lets-encrypt/.well-known/acme-challenge/" - local ngx_req = ngx.req - ngx_req.read_body() - local ret, data = pcall(decode, ngx_req.get_body_data()) - if not ret then - return self:ret(true, "json body decoding failed", HTTP_BAD_REQUEST) - end - execute("mkdir -p " .. acme_folder) - if self.ctx.bw.request_method == "POST" then - local file, err = open(acme_folder .. data.token, "w+") - if not file then - return self:ret(true, "can't write validation token : " .. err, HTTP_INTERNAL_SERVER_ERROR) - end - file:write(data.validation) - file:close() - return self:ret(true, "validation token written", HTTP_OK) - elseif self.ctx.bw.request_method == "DELETE" then - local ok, err = remove(acme_folder .. data.token) - if not ok then - return self:ret(true, "can't remove validation token : " .. err, HTTP_INTERNAL_SERVER_ERROR) - end - return self:ret(true, "validation token removed", HTTP_OK) - end - return self:ret(true, "unknown request", HTTP_NOT_FOUND) + debug_log(self.logger, "API endpoint called") + + if not match(self.ctx.bw.uri, "^/lets%-encrypt/challenge$") + or (self.ctx.bw.request_method ~= "POST" + and self.ctx.bw.request_method ~= "DELETE") + then + debug_log(self.logger, "API request does not match expected pattern") + return self:ret(false, "success") + end + + local acme_folder = "/var/tmp/bunkerweb/lets-encrypt/" .. + ".well-known/acme-challenge/" + local ngx_req = ngx.req + ngx_req.read_body() + local ret, data = pcall(decode, ngx_req.get_body_data()) + if not ret then + debug_log(self.logger, "Failed to decode JSON body") + return self:ret(true, "json body decoding failed", HTTP_BAD_REQUEST) + end + + debug_log(self.logger, "Creating ACME challenge directory: " .. + acme_folder) + execute("mkdir -p " .. acme_folder) + + if self.ctx.bw.request_method == "POST" then + debug_log(self.logger, "Processing POST request for token: " .. + data.token) + + local file, err = open(acme_folder .. data.token, "w+") + if not file then + return self:ret(true, "can't write validation token : " .. err, + HTTP_INTERNAL_SERVER_ERROR) + end + file:write(data.validation) + file:close() + + debug_log(self.logger, "Validation token written successfully") + return self:ret(true, "validation token written", HTTP_OK) + + elseif self.ctx.bw.request_method == "DELETE" then + debug_log(self.logger, "Processing DELETE request for token: " .. + data.token) + + local ok, err = remove(acme_folder .. data.token) + if not ok then + return self:ret(true, "can't remove validation token : " .. err, + HTTP_INTERNAL_SERVER_ERROR) + end + + debug_log(self.logger, "Validation token removed successfully") + return self:ret(true, "validation token removed", HTTP_OK) + end + + debug_log(self.logger, "Unknown request method") + return self:ret(true, "unknown request", HTTP_NOT_FOUND) end -return letsencrypt +return letsencrypt \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/actions.py b/src/common/core/letsencrypt/ui/actions.py index 7f261833cf..b1edff73af 100644 --- a/src/common/core/letsencrypt/ui/actions.py +++ b/src/common/core/letsencrypt/ui/actions.py @@ -15,27 +15,27 @@ from cryptography.hazmat.primitives import hashes +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + def extract_cache(folder_path, cache_files): # Extract Let's Encrypt cache files to specified folder path. - # - # Args: - # folder_path (Path): Destination folder path for extraction - # cache_files (list): List of cache file dictionaries containing - # file data to extract logger = getLogger("UI") is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - logger.debug(f"Starting cache extraction to {folder_path}") - logger.debug(f"Processing {len(cache_files)} cache files") - logger.debug(f"Target folder exists: {folder_path.exists()}") + debug_log(logger, f"Starting cache extraction to {folder_path}") + debug_log(logger, f"Processing {len(cache_files)} cache files") + debug_log(logger, f"Target folder exists: {folder_path.exists()}") folder_path.mkdir(parents=True, exist_ok=True) - if is_debug: - logger.debug(f"Created directory structure: {folder_path}") - logger.debug(f"Directory permissions: " - f"{oct(folder_path.stat().st_mode)}") + debug_log(logger, f"Created directory structure: {folder_path}") + debug_log(logger, + f"Directory permissions: {oct(folder_path.stat().st_mode)}") extracted_files = 0 total_bytes = 0 @@ -44,92 +44,84 @@ def extract_cache(folder_path, cache_files): file_name = cache_file.get("file_name", "unknown") file_data = cache_file.get("data", b"") - if is_debug: - logger.debug(f"Examining cache file {i+1}/{len(cache_files)}: " - f"{file_name}") - logger.debug(f"File size: {len(file_data)} bytes") + debug_log(logger, f"Examining cache file {i+1}/{len(cache_files)}: " + f"{file_name}") + debug_log(logger, f"File size: {len(file_data)} bytes") if (cache_file["file_name"].endswith(".tgz") and cache_file["file_name"].startswith("folder:")): - if is_debug: - logger.debug(f"Processing archive: {cache_file['file_name']}") - logger.debug(f"Archive size: {len(cache_file['data'])} bytes") + debug_log(logger, + f"Processing archive: {cache_file['file_name']}") + debug_log(logger, + f"Archive size: {len(cache_file['data'])} bytes") try: with tar_open(fileobj=BytesIO(cache_file["data"]), mode="r:gz") as tar: members = tar.getmembers() - if is_debug: - logger.debug(f"Archive contains {len(members)} " - f"members") - # Show first few members - for j, member in enumerate(members[:5]): - logger.debug( - f" Member {j+1}: {member.name} " - f"({member.size} bytes, " - f"{'dir' if member.isdir() else 'file'})" - ) - if len(members) > 5: - logger.debug(f" ... and {len(members) - 5} " - f"more members") + debug_log(logger, + f"Archive contains {len(members)} members") + # Show first few members + for j, member in enumerate(members[:5]): + debug_log(logger, + f" Member {j+1}: {member.name} " + f"({member.size} bytes, " + f"{'dir' if member.isdir() else 'file'})") + if len(members) > 5: + debug_log(logger, + f" ... and {len(members) - 5} more members") try: tar.extractall(folder_path, filter="fully_trusted") - if is_debug: - logger.debug("Extraction completed with " - "fully_trusted filter") + debug_log(logger, + "Extraction completed with fully_trusted filter") except TypeError: # Fallback for older Python versions without filter - if is_debug: - logger.debug("Using fallback extraction without " - "filter") + debug_log(logger, + "Using fallback extraction without filter") tar.extractall(folder_path) extracted_files += 1 total_bytes += len(cache_file['data']) - if is_debug: - logger.debug(f"Successfully extracted " - f"{cache_file['file_name']}") - logger.debug(f"Extracted {len(members)} items from " - f"archive") + debug_log(logger, + f"Successfully extracted {cache_file['file_name']}") + debug_log(logger, + f"Extracted {len(members)} items from archive") except Exception as e: logger.error(f"Failed to extract {cache_file['file_name']}: " f"{e}") - if is_debug: - logger.debug(f"Extraction error details: " - f"{format_exc()}") + debug_log(logger, f"Extraction error details: {format_exc()}") else: - if is_debug: - logger.debug(f"Skipping non-archive file: {file_name}") + debug_log(logger, f"Skipping non-archive file: {file_name}") - if is_debug: - logger.debug(f"Cache extraction completed:") - logger.debug(f" - Files processed: {len(cache_files)}") - logger.debug(f" - Archives extracted: {extracted_files}") - logger.debug(f" - Total bytes processed: {total_bytes}") + debug_log(logger, "Cache extraction completed:") + debug_log(logger, f" - Files processed: {len(cache_files)}") + debug_log(logger, f" - Archives extracted: {extracted_files}") + debug_log(logger, f" - Total bytes processed: {total_bytes}") + + # List final directory contents + if folder_path.exists(): + all_items = list(folder_path.rglob("*")) + files = [item for item in all_items if item.is_file()] + dirs = [item for item in all_items if item.is_dir()] - # List final directory contents - if folder_path.exists(): - all_items = list(folder_path.rglob("*")) - files = [item for item in all_items if item.is_file()] - dirs = [item for item in all_items if item.is_dir()] - - logger.debug(f"Final directory structure:") - logger.debug(f" - Total items: {len(all_items)}") - logger.debug(f" - Files: {len(files)}") - logger.debug(f" - Directories: {len(dirs)}") - - # Show some example files - for i, file_item in enumerate(files[:5]): - rel_path = file_item.relative_to(folder_path) - logger.debug(f" File {i+1}: {rel_path} " - f"({file_item.stat().st_size} bytes)") - if len(files) > 5: - logger.debug(f" ... and {len(files) - 5} more files") + debug_log(logger, "Final directory structure:") + debug_log(logger, f" - Total items: {len(all_items)}") + debug_log(logger, f" - Files: {len(files)}") + debug_log(logger, f" - Directories: {len(dirs)}") + + # Show some example files + for i, file_item in enumerate(files[:5]): + rel_path = file_item.relative_to(folder_path) + debug_log(logger, + f" File {i+1}: {rel_path} " + f"({file_item.stat().st_size} bytes)") + if len(files) > 5: + debug_log(logger, f" ... and {len(files) - 5} more files") def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: @@ -138,21 +130,11 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Parses Let's Encrypt certificate files and renewal configurations # to extract detailed certificate information including validity dates, # issuer information, and configuration details. - # - # Args: - # folder_paths (Tuple[Path, ...]): Tuple of folder paths containing - # certificate data to process - # - # Returns: - # dict: Dictionary containing lists of certificate information - # with keys for domain, common_name, issuer, validity dates, - # and other certificate metadata logger = getLogger("UI") is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - logger.debug(f"Retrieving certificate info from " - f"{len(folder_paths)} folder paths") + debug_log(logger, + f"Retrieving certificate info from {len(folder_paths)} folder paths") certificates = { "domain": [], @@ -173,24 +155,22 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: total_certs_processed = 0 for folder_idx, folder_path in enumerate(folder_paths): - if is_debug: - logger.debug(f"Processing folder {folder_idx + 1}/" - f"{len(folder_paths)}: {folder_path}") + debug_log(logger, + f"Processing folder {folder_idx + 1}/{len(folder_paths)}: " + f"{folder_path}") cert_files = list(folder_path.joinpath("live").glob("*/fullchain.pem")) - if is_debug: - logger.debug(f"Found {len(cert_files)} certificate files in " - f"{folder_path}") + debug_log(logger, f"Found {len(cert_files)} certificate files in " + f"{folder_path}") for cert_file in cert_files: domain = cert_file.parent.name certificates["domain"].append(domain) total_certs_processed += 1 - if is_debug: - logger.debug(f"Processing certificate " - f"{total_certs_processed}: {domain}") + debug_log(logger, + f"Processing certificate {total_certs_processed}: {domain}") # Initialize default certificate information cert_info = { @@ -211,28 +191,25 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: # Parse the certificate file try: - if is_debug: - logger.debug(f"Loading X.509 certificate from " - f"{cert_file}") - logger.debug(f"Certificate file size: " - f"{cert_file.stat().st_size} bytes") + debug_log(logger, + f"Loading X.509 certificate from {cert_file}") + debug_log(logger, + f"Certificate file size: {cert_file.stat().st_size} bytes") cert_bytes = cert_file.read_bytes() - if is_debug: - logger.debug(f"Read {len(cert_bytes)} bytes from " - f"certificate file") - logger.debug(f"Certificate data preview: " - f"{cert_bytes[:100]}...") + debug_log(logger, + f"Read {len(cert_bytes)} bytes from certificate file") + debug_log(logger, + f"Certificate data preview: {cert_bytes[:100]}...") cert = x509.load_pem_x509_certificate( cert_bytes, default_backend() ) - if is_debug: - logger.debug(f"Successfully loaded certificate for " - f"{domain}") - logger.debug(f"Certificate version: {cert.version}") - logger.debug(f"Certificate serial: {cert.serial_number}") + debug_log(logger, + f"Successfully loaded certificate for {domain}") + debug_log(logger, f"Certificate version: {cert.version}") + debug_log(logger, f"Certificate serial: {cert.serial_number}") # Extract subject (Common Name) subject = cert.subject.get_attributes_for_oid( @@ -240,14 +217,12 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: ) if subject: cert_info["common_name"] = subject[0].value - if is_debug: - logger.debug(f"Certificate CN: " - f"{cert_info['common_name']}") + debug_log(logger, + f"Certificate CN: {cert_info['common_name']}") else: - if is_debug: - logger.debug("No Common Name found in certificate " - "subject") - logger.debug(f"Full subject: {cert.subject}") + debug_log(logger, + "No Common Name found in certificate subject") + debug_log(logger, f"Full subject: {cert.subject}") # Extract issuer (Certificate Authority) issuer = cert.issuer.get_attributes_for_oid( @@ -255,14 +230,12 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: ) if issuer: cert_info["issuer"] = issuer[0].value - if is_debug: - logger.debug(f"Certificate issuer: " - f"{cert_info['issuer']}") + debug_log(logger, + f"Certificate issuer: {cert_info['issuer']}") else: - if is_debug: - logger.debug("No Common Name found in certificate " - "issuer") - logger.debug(f"Full issuer: {cert.issuer}") + debug_log(logger, + "No Common Name found in certificate issuer") + debug_log(logger, f"Full issuer: {cert.issuer}") # Extract validity period cert_info["valid_from"] = ( @@ -272,16 +245,15 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: cert.not_valid_after.strftime("%d-%m-%Y %H:%M:%S UTC") ) - if is_debug: - logger.debug(f"Certificate validity: " - f"{cert_info['valid_from']} to " - f"{cert_info['valid_to']}") - # Check if certificate is currently valid - from datetime import datetime, timezone - now = datetime.now(timezone.utc) - is_valid = (cert.not_valid_before <= now <= - cert.not_valid_after) - logger.debug(f"Certificate currently valid: {is_valid}") + debug_log(logger, + f"Certificate validity: {cert_info['valid_from']} to " + f"{cert_info['valid_to']}") + # Check if certificate is currently valid + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + is_valid = (cert.not_valid_before <= now <= + cert.not_valid_after) + debug_log(logger, f"Certificate currently valid: {is_valid}") # Extract serial number cert_info["serial_number"] = str(cert.serial_number) @@ -290,9 +262,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: fingerprint_bytes = cert.fingerprint(hashes.SHA256()) cert_info["fingerprint"] = fingerprint_bytes.hex() - if is_debug: - logger.debug(f"Certificate fingerprint: " - f"{cert_info['fingerprint']}") + debug_log(logger, + f"Certificate fingerprint: {cert_info['fingerprint']}") # Extract version cert_info["version"] = cert.version.name @@ -312,48 +283,44 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: if ocsp_urls: cert_info["ocsp_support"] = "Yes" - if is_debug: - logger.debug(f"OCSP URLs found for {domain}: " - f"{ocsp_urls}") + debug_log(logger, + f"OCSP URLs found for {domain}: {ocsp_urls}") else: cert_info["ocsp_support"] = "No" - if is_debug: - logger.debug(f"AIA extension found for {domain} " - f"but no OCSP URLs") + debug_log(logger, + f"AIA extension found for {domain} but no OCSP URLs") except x509.ExtensionNotFound: cert_info["ocsp_support"] = "No" - if is_debug: - logger.debug(f"No Authority Information Access " - f"extension found for {domain}") + debug_log(logger, + f"No Authority Information Access extension found for " + f"{domain}") except Exception as ocsp_error: cert_info["ocsp_support"] = "Unknown" - if is_debug: - logger.debug(f"Error checking OCSP support for " - f"{domain}: {ocsp_error}") + debug_log(logger, + f"Error checking OCSP support for {domain}: " + f"{ocsp_error}") - if is_debug: - logger.debug(f"Certificate processing completed for " - f"{domain}") - logger.debug(f" - Serial: {cert_info['serial_number']}") - logger.debug(f" - Version: {cert_info['version']}") - logger.debug(f" - Subject: {cert_info['common_name']}") - logger.debug(f" - Issuer: {cert_info['issuer']}") - logger.debug(f" - OCSP Support: " - f"{cert_info['ocsp_support']}") + debug_log(logger, + f"Certificate processing completed for {domain}") + debug_log(logger, f" - Serial: {cert_info['serial_number']}") + debug_log(logger, f" - Version: {cert_info['version']}") + debug_log(logger, f" - Subject: {cert_info['common_name']}") + debug_log(logger, f" - Issuer: {cert_info['issuer']}") + debug_log(logger, + f" - OCSP Support: {cert_info['ocsp_support']}") except BaseException as e: error_msg = (f"Error while parsing certificate {cert_file}: " f"{e}") logger.error(error_msg) - if is_debug: - logger.debug(f"Certificate parsing error details:") - logger.debug(f" - Error type: {type(e).__name__}") - logger.debug(f" - Error message: {str(e)}") - logger.debug(f" - Full traceback: {format_exc()}") - logger.debug(f" - Certificate file: {cert_file}") - logger.debug(f" - File exists: {cert_file.exists()}") - logger.debug(f" - File readable: {cert_file.is_file()}") + debug_log(logger, "Certificate parsing error details:") + debug_log(logger, f" - Error type: {type(e).__name__}") + debug_log(logger, f" - Error message: {str(e)}") + debug_log(logger, f" - Full traceback: {format_exc()}") + debug_log(logger, f" - Certificate file: {cert_file}") + debug_log(logger, f" - File exists: {cert_file.exists()}") + debug_log(logger, f" - File readable: {cert_file.is_file()}") # Parse the renewal configuration file try: @@ -361,9 +328,8 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: f"{domain}.conf") if renewal_file.exists(): - if is_debug: - logger.debug(f"Processing renewal configuration: " - f"{renewal_file}") + debug_log(logger, + f"Processing renewal configuration: {renewal_file}") with renewal_file.open("r") as f: for line_num, line in enumerate(f, 1): @@ -392,40 +358,38 @@ def retrieve_certificates_info(folder_paths: Tuple[Path, ...]) -> dict: line.split(" = ")[1].strip() ) - if is_debug: - logger.debug(f"Renewal config parsed - Profile: " - f"{cert_info['preferred_profile']}, " - f"Challenge: {cert_info['challenge']}, " - f"Key type: {cert_info['key_type']}") + debug_log(logger, + f"Renewal config parsed - Profile: " + f"{cert_info['preferred_profile']}, " + f"Challenge: {cert_info['challenge']}, " + f"Key type: {cert_info['key_type']}") else: - if is_debug: - logger.debug(f"No renewal configuration found for " - f"{domain}") + debug_log(logger, + f"No renewal configuration found for {domain}") except BaseException as e: error_msg = (f"Error while parsing renewal configuration " f"{renewal_file}: {e}") logger.error(error_msg) - if is_debug: - logger.debug(f"Renewal config parsing error: " - f"{format_exc()}") + debug_log(logger, f"Renewal config parsing error: " + f"{format_exc()}") # Append all certificate information to the results for key in cert_info: certificates[key].append(cert_info[key]) - if is_debug: - logger.debug(f"Certificate retrieval complete. Processed " - f"{total_certs_processed} certificates from " - f"{len(folder_paths)} folders") - - # Summary of OCSP support - ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} - for ocsp_status in certificates.get('ocsp_support', []): - ocsp_support_counts[ocsp_status] = ( - ocsp_support_counts.get(ocsp_status, 0) + 1 - ) - logger.debug(f"OCSP support summary: {ocsp_support_counts}") + debug_log(logger, + f"Certificate retrieval complete. Processed " + f"{total_certs_processed} certificates from {len(folder_paths)} " + f"folders") + + # Summary of OCSP support + ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} + for ocsp_status in certificates.get('ocsp_support', []): + ocsp_support_counts[ocsp_status] = ( + ocsp_support_counts.get(ocsp_status, 0) + 1 + ) + debug_log(logger, f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -437,21 +401,10 @@ def pre_render(app, *args, **kwargs): # prepares the data structure for rendering in the web interface. # Handles extraction of cache files, certificate parsing, and error # handling for the certificate management interface. - # - # Args: - # app: Flask application instance - # *args: Variable length argument list - # **kwargs: Keyword arguments containing database connection and - # other configuration options - # - # Returns: - # dict: Dictionary containing certificate data, display configuration, - # and any error information for the UI template logger = getLogger("UI") is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - logger.debug("Starting pre-render for Let's Encrypt certificates") + debug_log(logger, "Starting pre-render for Let's Encrypt certificates") # Initialize return structure with default values ret = { @@ -485,94 +438,86 @@ def pre_render(app, *args, **kwargs): folder_path = None try: - if is_debug: - logger.debug("Starting Let's Encrypt data retrieval process") - logger.debug(f"Database connection available: " - f"{'db' in kwargs}") - logger.debug(f"Root folder: {root_folder}") + debug_log(logger, "Starting Let's Encrypt data retrieval process") + debug_log(logger, f"Database connection available: {'db' in kwargs}") + debug_log(logger, f"Root folder: {root_folder}") # Retrieve cache files from database - if is_debug: - logger.debug("Fetching cache files from database for job: " - "certbot-renew") + debug_log(logger, + "Fetching cache files from database for job: certbot-renew") regular_cache_files = kwargs["db"].get_jobs_cache_files( job_name="certbot-renew" ) - if is_debug: - logger.debug(f"Retrieved {len(regular_cache_files)} cache files") - for i, cache_file in enumerate(regular_cache_files): - file_name = cache_file.get("file_name", "unknown") - file_size = len(cache_file.get("data", b"")) - logger.debug(f" Cache file {i+1}: {file_name} " - f"({file_size} bytes)") + debug_log(logger, f"Retrieved {len(regular_cache_files)} cache files") + for i, cache_file in enumerate(regular_cache_files): + file_name = cache_file.get("file_name", "unknown") + file_size = len(cache_file.get("data", b"")) + debug_log(logger, + f" Cache file {i+1}: {file_name} ({file_size} bytes)") # Create unique temporary folder for extraction folder_uuid = str(uuid4()) folder_path = root_folder.joinpath("letsencrypt", folder_uuid) regular_le_folder = folder_path.joinpath("regular") - if is_debug: - logger.debug(f"Using temporary folder UUID: {folder_uuid}") - logger.debug(f"Temporary folder path: {folder_path}") - logger.debug(f"Regular LE folder: {regular_le_folder}") + debug_log(logger, f"Using temporary folder UUID: {folder_uuid}") + debug_log(logger, f"Temporary folder path: {folder_path}") + debug_log(logger, f"Regular LE folder: {regular_le_folder}") # Extract cache files to temporary location - if is_debug: - logger.debug("Starting cache file extraction") + debug_log(logger, "Starting cache file extraction") extract_cache(regular_le_folder, regular_cache_files) - if is_debug: - logger.debug("Cache extraction completed, starting certificate " - "parsing") + debug_log(logger, + "Cache extraction completed, starting certificate parsing") # Parse certificates and retrieve information cert_data = retrieve_certificates_info((regular_le_folder,)) cert_count = len(cert_data.get("domain", [])) - if is_debug: - logger.debug(f"Certificate parsing completed") - logger.debug(f"Total certificates processed: {cert_count}") - logger.debug(f"Certificate data keys: {list(cert_data.keys())}") - - # Log sample certificate data (first certificate if available) - if cert_count > 0: - logger.debug("Sample certificate data (first certificate):") - for key in cert_data: - value = cert_data[key][0] if cert_data[key] else "None" - if key == "ocsp_support": - logger.debug(f" {key}: {value} (OCSP support " - f"detected)") - else: - logger.debug(f" {key}: {value}") + debug_log(logger, "Certificate parsing completed") + debug_log(logger, f"Total certificates processed: {cert_count}") + debug_log(logger, f"Certificate data keys: {list(cert_data.keys())}") + + # Log sample certificate data (first certificate if available) + if cert_count > 0: + debug_log(logger, + "Sample certificate data (first certificate):") + for key in cert_data: + value = cert_data[key][0] if cert_data[key] else "None" + if key == "ocsp_support": + debug_log(logger, + f" {key}: {value} (OCSP support detected)") + else: + debug_log(logger, f" {key}: {value}") ret["list_certificates"]["data"] = cert_data logger.info(f"Pre-render completed successfully with {cert_count} " f"certificates") - if is_debug: - logger.debug(f"Return data structure keys: {list(ret.keys())}") - logger.debug(f"Certificate list structure: " - f"{list(ret['list_certificates'].keys())}") + debug_log(logger, f"Return data structure keys: {list(ret.keys())}") + debug_log(logger, + f"Certificate list structure: " + f"{list(ret['list_certificates'].keys())}") except BaseException as e: error_msg = f"Failed to get Let's Encrypt certificates: {e}" logger.error(error_msg) - if is_debug: - logger.debug(f"Pre-render error occurred:") - logger.debug(f" - Error type: {type(e).__name__}") - logger.debug(f" - Error message: {str(e)}") - logger.debug(f" - Error traceback: {format_exc()}") - logger.debug(f" - kwargs keys: " - f"{list(kwargs.keys()) if kwargs else 'None'}") - if "db" in kwargs: - logger.debug(f" - Database object type: " - f"{type(kwargs['db'])}") + debug_log(logger, "Pre-render error occurred:") + debug_log(logger, f" - Error type: {type(e).__name__}") + debug_log(logger, f" - Error message: {str(e)}") + debug_log(logger, f" - Error traceback: {format_exc()}") + debug_log(logger, + f" - kwargs keys: {list(kwargs.keys()) if kwargs else 'None'}") + if "db" in kwargs: + debug_log(logger, + f" - Database object type: {type(kwargs['db'])}") ret["error"] = {"message": str(e)} @@ -580,20 +525,17 @@ def pre_render(app, *args, **kwargs): # Clean up temporary files if folder_path and folder_path.exists(): try: - if is_debug: - logger.debug(f"Cleaning up temporary folder: " - f"{root_folder}") + debug_log(logger, + f"Cleaning up temporary folder: {root_folder}") rmtree(root_folder, ignore_errors=True) - if is_debug: - logger.debug("Temporary folder cleanup completed") + debug_log(logger, "Temporary folder cleanup completed") except Exception as cleanup_error: logger.warning(f"Failed to clean up temporary folder " f"{root_folder}: {cleanup_error}") - if is_debug: - logger.debug("Pre-render function completed") + debug_log(logger, "Pre-render function completed") return ret \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py index 0d8c8c318f..f1f797b7b7 100644 --- a/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py +++ b/src/common/core/letsencrypt/ui/blueprints/letsencrypt.py @@ -38,6 +38,13 @@ DEPS_PATH = join(sep, "usr", "share", "bunkerweb", "deps", "python") +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + def download_certificates(): # Download and extract Let's Encrypt certificates from database cache. # @@ -45,87 +52,73 @@ def download_certificates(): # to the local data path for processing. is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - LOGGER.debug(f"Starting certificate download process") - LOGGER.debug(f"Target directory: {DATA_PATH}") - LOGGER.debug(f"Cache directory: {LE_CACHE_DIR}") + debug_log(LOGGER, "Starting certificate download process") + debug_log(LOGGER, f"Target directory: {DATA_PATH}") + debug_log(LOGGER, f"Cache directory: {LE_CACHE_DIR}") # Clean up and create fresh directory if Path(DATA_PATH).exists(): - if is_debug: - LOGGER.debug(f"Removing existing directory: {DATA_PATH}") + debug_log(LOGGER, f"Removing existing directory: {DATA_PATH}") rmtree(DATA_PATH, ignore_errors=True) - if is_debug: - LOGGER.debug(f"Creating directory structure: {DATA_PATH}") + debug_log(LOGGER, f"Creating directory structure: {DATA_PATH}") Path(DATA_PATH).mkdir(parents=True, exist_ok=True) - if is_debug: - LOGGER.debug("Fetching cache files from database") + debug_log(LOGGER, "Fetching cache files from database") cache_files = DB.get_jobs_cache_files(job_name="certbot-renew") - if is_debug: - LOGGER.debug(f"Retrieved {len(cache_files)} cache files") - for i, cache_file in enumerate(cache_files): - LOGGER.debug(f"Cache file {i+1}: {cache_file['file_name']} " - f"({len(cache_file.get('data', b''))} bytes)") + debug_log(LOGGER, f"Retrieved {len(cache_files)} cache files") + for i, cache_file in enumerate(cache_files): + debug_log(LOGGER, f"Cache file {i+1}: {cache_file['file_name']} " + f"({len(cache_file.get('data', b''))} bytes)") extracted_count = 0 for cache_file in cache_files: if (cache_file["file_name"].endswith(".tgz") and cache_file["file_name"].startswith("folder:")): - if is_debug: - LOGGER.debug(f"Extracting cache file: " - f"{cache_file['file_name']}") - LOGGER.debug(f"File size: {len(cache_file['data'])} bytes") + debug_log(LOGGER, + f"Extracting cache file: {cache_file['file_name']}") + debug_log(LOGGER, f"File size: {len(cache_file['data'])} bytes") try: with tar_open(fileobj=BytesIO(cache_file["data"]), mode="r:gz") as tar: member_count = len(tar.getmembers()) - if is_debug: - LOGGER.debug(f"Archive contains {member_count} " - f"members") + debug_log(LOGGER, + f"Archive contains {member_count} members") try: tar.extractall(DATA_PATH, filter="fully_trusted") - if is_debug: - LOGGER.debug("Extraction completed with " - "fully_trusted filter") + debug_log(LOGGER, + "Extraction completed with fully_trusted filter") except TypeError: - if is_debug: - LOGGER.debug("Falling back to extraction without " - "filter") + debug_log(LOGGER, + "Falling back to extraction without filter") tar.extractall(DATA_PATH) extracted_count += 1 - if is_debug: - LOGGER.debug(f"Successfully extracted " - f"{cache_file['file_name']}") + debug_log(LOGGER, + f"Successfully extracted {cache_file['file_name']}") except Exception as e: LOGGER.error(f"Failed to extract {cache_file['file_name']}: " f"{e}") - if is_debug: - LOGGER.debug(f"Extraction error details: {format_exc()}") + debug_log(LOGGER, f"Extraction error details: {format_exc()}") else: - if is_debug: - LOGGER.debug(f"Skipping non-matching file: " - f"{cache_file['file_name']}") - - if is_debug: - LOGGER.debug(f"Certificate download completed: {extracted_count} " - f"files extracted") - # List extracted directory contents - if Path(DATA_PATH).exists(): - contents = list(Path(DATA_PATH).rglob("*")) - LOGGER.debug(f"Extracted directory contains {len(contents)} " - f"items") - for item in contents[:10]: # Show first 10 items - LOGGER.debug(f" - {item}") - if len(contents) > 10: - LOGGER.debug(f" ... and {len(contents) - 10} more items") + debug_log(LOGGER, + f"Skipping non-matching file: {cache_file['file_name']}") + + debug_log(LOGGER, + f"Certificate download completed: {extracted_count} files extracted") + # List extracted directory contents + if Path(DATA_PATH).exists(): + contents = list(Path(DATA_PATH).rglob("*")) + debug_log(LOGGER, f"Extracted directory contains {len(contents)} items") + for item in contents[:10]: # Show first 10 items + debug_log(LOGGER, f" - {item}") + if len(contents) > 10: + debug_log(LOGGER, f" ... and {len(contents) - 10} more items") def retrieve_certificates(): @@ -134,14 +127,9 @@ def retrieve_certificates(): # Downloads certificates from cache and parses both the certificate # files and renewal configuration to extract comprehensive certificate # information. - # - # Returns: - # dict: Dictionary containing lists of certificate information - # including domain, issuer, validity dates, etc. is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - LOGGER.debug("Starting certificate retrieval") + debug_log(LOGGER, "Starting certificate retrieval") download_certificates() @@ -164,19 +152,17 @@ def retrieve_certificates(): cert_files = list(Path(DATA_PATH).joinpath("live").glob("*/fullchain.pem")) - if is_debug: - LOGGER.debug(f"Processing {len(cert_files)} certificate files") + debug_log(LOGGER, f"Processing {len(cert_files)} certificate files") for cert_file in cert_files: domain = cert_file.parent.name certificates["domain"].append(domain) - if is_debug: - LOGGER.debug(f"Processing certificate " - f"{len(certificates['domain'])}: {domain}") - LOGGER.debug(f"Certificate file path: {cert_file}") - LOGGER.debug(f"Certificate file size: " - f"{cert_file.stat().st_size} bytes") + debug_log(LOGGER, + f"Processing certificate {len(certificates['domain'])}: {domain}") + debug_log(LOGGER, f"Certificate file path: {cert_file}") + debug_log(LOGGER, + f"Certificate file size: {cert_file.stat().st_size} bytes") cert_info = { "common_name": "Unknown", @@ -195,49 +181,44 @@ def retrieve_certificates(): } try: - if is_debug: - LOGGER.debug(f"Loading X.509 certificate from {cert_file}") + debug_log(LOGGER, f"Loading X.509 certificate from {cert_file}") cert_data = cert_file.read_bytes() - if is_debug: - LOGGER.debug(f"Certificate data length: {len(cert_data)} " - f"bytes") - LOGGER.debug(f"Certificate starts with: {cert_data[:50]}") + debug_log(LOGGER, + f"Certificate data length: {len(cert_data)} bytes") + debug_log(LOGGER, + f"Certificate starts with: {cert_data[:50]}") cert = x509.load_pem_x509_certificate( cert_data, default_backend() ) - if is_debug: - LOGGER.debug(f"Successfully loaded certificate for {domain}") - LOGGER.debug(f"Certificate subject: {cert.subject}") - LOGGER.debug(f"Certificate issuer: {cert.issuer}") + debug_log(LOGGER, + f"Successfully loaded certificate for {domain}") + debug_log(LOGGER, f"Certificate subject: {cert.subject}") + debug_log(LOGGER, f"Certificate issuer: {cert.issuer}") subject = cert.subject.get_attributes_for_oid( x509.NameOID.COMMON_NAME ) if subject: cert_info["common_name"] = subject[0].value - if is_debug: - LOGGER.debug(f"Certificate CN extracted: " - f"{cert_info['common_name']}") + debug_log(LOGGER, + f"Certificate CN extracted: {cert_info['common_name']}") else: - if is_debug: - LOGGER.debug("No Common Name found in certificate " - "subject") + debug_log(LOGGER, + "No Common Name found in certificate subject") issuer = cert.issuer.get_attributes_for_oid( x509.NameOID.COMMON_NAME ) if issuer: cert_info["issuer"] = issuer[0].value - if is_debug: - LOGGER.debug(f"Certificate issuer extracted: " - f"{cert_info['issuer']}") + debug_log(LOGGER, + f"Certificate issuer extracted: {cert_info['issuer']}") else: - if is_debug: - LOGGER.debug("No Common Name found in certificate " - "issuer") + debug_log(LOGGER, + "No Common Name found in certificate issuer") cert_info["valid_from"] = ( cert.not_valid_before.astimezone().isoformat() @@ -246,10 +227,9 @@ def retrieve_certificates(): cert.not_valid_after.astimezone().isoformat() ) - if is_debug: - LOGGER.debug(f"Certificate validity period: " - f"{cert_info['valid_from']} to " - f"{cert_info['valid_to']}") + debug_log(LOGGER, + f"Certificate validity period: {cert_info['valid_from']} to " + f"{cert_info['valid_to']}") cert_info["serial_number"] = str(cert.serial_number) cert_info["fingerprint"] = cert.fingerprint(hashes.SHA256()).hex() @@ -270,48 +250,42 @@ def retrieve_certificates(): if ocsp_urls: cert_info["ocsp_support"] = "Yes" - if is_debug: - LOGGER.debug(f"OCSP URLs found: {ocsp_urls}") + debug_log(LOGGER, f"OCSP URLs found: {ocsp_urls}") else: cert_info["ocsp_support"] = "No" - if is_debug: - LOGGER.debug("AIA extension found but no OCSP URLs") + debug_log(LOGGER, + "AIA extension found but no OCSP URLs") except x509.ExtensionNotFound: cert_info["ocsp_support"] = "No" - if is_debug: - LOGGER.debug("No Authority Information Access extension " - "found") + debug_log(LOGGER, + "No Authority Information Access extension found") except Exception as ocsp_error: cert_info["ocsp_support"] = "Unknown" - if is_debug: - LOGGER.debug(f"Error checking OCSP support: " - f"{ocsp_error}") + debug_log(LOGGER, f"Error checking OCSP support: {ocsp_error}") - if is_debug: - LOGGER.debug(f"Certificate details extracted:") - LOGGER.debug(f" - Serial: {cert_info['serial_number']}") - LOGGER.debug(f" - Fingerprint: " - f"{cert_info['fingerprint'][:16]}...") - LOGGER.debug(f" - Version: {cert_info['version']}") - LOGGER.debug(f" - OCSP Support: {cert_info['ocsp_support']}") + debug_log(LOGGER, "Certificate details extracted:") + debug_log(LOGGER, f" - Serial: {cert_info['serial_number']}") + debug_log(LOGGER, + f" - Fingerprint: {cert_info['fingerprint'][:16]}...") + debug_log(LOGGER, f" - Version: {cert_info['version']}") + debug_log(LOGGER, + f" - OCSP Support: {cert_info['ocsp_support']}") except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while parsing certificate {cert_file}: {e}") - if is_debug: - LOGGER.debug(f"Certificate parsing failed for {domain}: " - f"{str(e)}") - LOGGER.debug(f"Error type: {type(e).__name__}") + debug_log(LOGGER, + f"Certificate parsing failed for {domain}: {str(e)}") + debug_log(LOGGER, f"Error type: {type(e).__name__}") try: renewal_file = Path(DATA_PATH).joinpath("renewal", f"{domain}.conf") if renewal_file.exists(): - if is_debug: - LOGGER.debug(f"Processing renewal file: {renewal_file}") - LOGGER.debug(f"Renewal file size: " - f"{renewal_file.stat().st_size} bytes") + debug_log(LOGGER, f"Processing renewal file: {renewal_file}") + debug_log(LOGGER, + f"Renewal file size: {renewal_file.stat().st_size} bytes") config_lines_processed = 0 with renewal_file.open("r") as f: @@ -322,82 +296,78 @@ def retrieve_certificates(): config_lines_processed += 1 if is_debug and line_num <= 10: # Debug first 10 lines - LOGGER.debug(f"Renewal config line {line_num}: " - f"{line}") + debug_log(LOGGER, + f"Renewal config line {line_num}: {line}") if line.startswith("preferred_profile = "): cert_info["preferred_profile"] = ( line.split(" = ")[1].strip() ) - if is_debug: - LOGGER.debug(f"Found preferred_profile: " - f"{cert_info['preferred_profile']}") + debug_log(LOGGER, + f"Found preferred_profile: " + f"{cert_info['preferred_profile']}") elif line.startswith("pref_challs = "): challenges = line.split(" = ")[1].strip() cert_info["challenge"] = challenges.split(",")[0] - if is_debug: - LOGGER.debug(f"Found challenge: " - f"{cert_info['challenge']} " - f"(from {challenges})") + debug_log(LOGGER, + f"Found challenge: {cert_info['challenge']} " + f"(from {challenges})") elif line.startswith("authenticator = "): cert_info["authenticator"] = ( line.split(" = ")[1].strip() ) - if is_debug: - LOGGER.debug(f"Found authenticator: " - f"{cert_info['authenticator']}") + debug_log(LOGGER, + f"Found authenticator: " + f"{cert_info['authenticator']}") elif line.startswith("server = "): cert_info["issuer_server"] = ( line.split(" = ")[1].strip() ) - if is_debug: - LOGGER.debug(f"Found issuer_server: " - f"{cert_info['issuer_server']}") + debug_log(LOGGER, + f"Found issuer_server: " + f"{cert_info['issuer_server']}") elif line.startswith("key_type = "): cert_info["key_type"] = ( line.split(" = ")[1].strip() ) - if is_debug: - LOGGER.debug(f"Found key_type: " - f"{cert_info['key_type']}") + debug_log(LOGGER, + f"Found key_type: {cert_info['key_type']}") - if is_debug: - LOGGER.debug(f"Processed {config_lines_processed} " - f"configuration lines") - LOGGER.debug(f"Final renewal configuration for {domain}:") - LOGGER.debug(f" - Profile: " - f"{cert_info['preferred_profile']}") - LOGGER.debug(f" - Challenge: {cert_info['challenge']}") - LOGGER.debug(f" - Authenticator: " - f"{cert_info['authenticator']}") - LOGGER.debug(f" - Server: {cert_info['issuer_server']}") - LOGGER.debug(f" - Key type: {cert_info['key_type']}") + debug_log(LOGGER, + f"Processed {config_lines_processed} configuration lines") + debug_log(LOGGER, + f"Final renewal configuration for {domain}:") + debug_log(LOGGER, + f" - Profile: {cert_info['preferred_profile']}") + debug_log(LOGGER, f" - Challenge: {cert_info['challenge']}") + debug_log(LOGGER, + f" - Authenticator: {cert_info['authenticator']}") + debug_log(LOGGER, + f" - Server: {cert_info['issuer_server']}") + debug_log(LOGGER, f" - Key type: {cert_info['key_type']}") else: - if is_debug: - LOGGER.debug(f"No renewal file found for {domain} at " - f"{renewal_file}") + debug_log(LOGGER, + f"No renewal file found for {domain} at {renewal_file}") except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error(f"Error while parsing renewal configuration " f"{renewal_file}: {e}") - if is_debug: - LOGGER.debug(f"Renewal config parsing failed for {domain}: " - f"{str(e)}") - LOGGER.debug(f"Error type: {type(e).__name__}") + debug_log(LOGGER, + f"Renewal config parsing failed for {domain}: {str(e)}") + debug_log(LOGGER, f"Error type: {type(e).__name__}") for key in cert_info: certificates[key].append(cert_info[key]) - if is_debug: - LOGGER.debug(f"Retrieved {len(certificates['domain'])} certificates") - # Summary of OCSP support - ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} - for ocsp_status in certificates.get('ocsp_support', []): - ocsp_support_counts[ocsp_status] = ( - ocsp_support_counts.get(ocsp_status, 0) + 1 - ) - LOGGER.debug(f"OCSP support summary: {ocsp_support_counts}") + debug_log(LOGGER, f"Retrieved {len(certificates['domain'])} certificates") + # Summary of OCSP support + ocsp_support_counts = {"Yes": 0, "No": 0, "Unknown": 0} + for ocsp_status in certificates.get('ocsp_support', []): + ocsp_support_counts[ocsp_status] = ( + ocsp_support_counts.get(ocsp_status, 0) + 1 + ) + debug_log(LOGGER, f"OCSP support summary: {ocsp_support_counts}") return certificates @@ -406,13 +376,9 @@ def retrieve_certificates(): @login_required def letsencrypt_page(): # Render the Let's Encrypt certificates management page. - # - # Returns: - # str: Rendered HTML template for the Let's Encrypt page is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - LOGGER.debug("Rendering Let's Encrypt page") + debug_log(LOGGER, "Rendering Let's Encrypt page") return render_template("letsencrypt.html") @@ -425,23 +391,17 @@ def letsencrypt_fetch(): # # Retrieves and formats certificate information for display in the # DataTables interface. - # - # Returns: - # dict: JSON response containing certificate data, record counts, - # and draw number for DataTables is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - LOGGER.debug("Fetching certificates for DataTables") + debug_log(LOGGER, "Fetching certificates for DataTables") cert_list = [] try: certs = retrieve_certificates() - if is_debug: - LOGGER.debug(f"Retrieved certificates: " - f"{len(certs.get('domain', []))}") + debug_log(LOGGER, f"Retrieved certificates: " + f"{len(certs.get('domain', []))}") for i, domain in enumerate(certs.get("domain", [])): cert_data = { @@ -462,12 +422,11 @@ def letsencrypt_fetch(): } cert_list.append(cert_data) - if is_debug: - LOGGER.debug(f"Added certificate to list: {domain}") - LOGGER.debug(f" - OCSP Support: " - f"{cert_data['ocsp_support']}") - LOGGER.debug(f" - Challenge: {cert_data['challenge']}") - LOGGER.debug(f" - Key Type: {cert_data['key_type']}") + debug_log(LOGGER, f"Added certificate to list: {domain}") + debug_log(LOGGER, + f" - OCSP Support: {cert_data['ocsp_support']}") + debug_log(LOGGER, f" - Challenge: {cert_data['challenge']}") + debug_log(LOGGER, f" - Key Type: {cert_data['key_type']}") except BaseException as e: LOGGER.debug(format_exc()) @@ -480,9 +439,7 @@ def letsencrypt_fetch(): "draw": int(request.form.get("draw", 1)), } - if is_debug: - LOGGER.debug(f"Returning {len(cert_list)} certificates to " - f"DataTables") + debug_log(LOGGER, f"Returning {len(cert_list)} certificates to DataTables") return jsonify(response_data) @@ -496,20 +453,14 @@ def letsencrypt_delete(): # Removes the specified certificate using certbot and cleans up # associated files and directories. Updates the database cache # with the modified certificate data. - # - # Returns: - # dict: JSON response indicating success or failure of the - # deletion operation is_debug = getenv("LOG_LEVEL") == "debug" cert_name = request.json.get("cert_name") if not cert_name: - if is_debug: - LOGGER.debug("Certificate deletion request missing cert_name") + debug_log(LOGGER, "Certificate deletion request missing cert_name") return jsonify({"status": "ko", "message": "Missing cert_name"}), 400 - if is_debug: - LOGGER.debug(f"Starting deletion of certificate: {cert_name}") + debug_log(LOGGER, f"Starting deletion of certificate: {cert_name}") download_certificates() @@ -518,10 +469,9 @@ def letsencrypt_delete(): f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" ) - if is_debug: - LOGGER.debug(f"Running certbot delete for {cert_name}") - LOGGER.debug(f"Environment: PATH={env['PATH'][:100]}...") - LOGGER.debug(f"PYTHONPATH: {env['PYTHONPATH'][:100]}...") + debug_log(LOGGER, f"Running certbot delete for {cert_name}") + debug_log(LOGGER, f"Environment: PATH={env['PATH'][:100]}...") + debug_log(LOGGER, f"PYTHONPATH: {env['PYTHONPATH'][:100]}...") delete_proc = run( [ @@ -545,10 +495,9 @@ def letsencrypt_delete(): check=False, ) - if is_debug: - LOGGER.debug(f"Certbot delete return code: {delete_proc.returncode}") - if delete_proc.stdout: - LOGGER.debug(f"Certbot output: {delete_proc.stdout}") + debug_log(LOGGER, f"Certbot delete return code: {delete_proc.returncode}") + if delete_proc.stdout: + debug_log(LOGGER, f"Certbot output: {delete_proc.stdout}") if delete_proc.returncode == 0: LOGGER.info(f"Successfully deleted certificate {cert_name}") @@ -559,8 +508,7 @@ def letsencrypt_delete(): renewal_file = Path(DATA_PATH).joinpath("renewal", f"{cert_name}.conf") - if is_debug: - LOGGER.debug(f"Cleaning up directories for {cert_name}") + debug_log(LOGGER, f"Cleaning up directories for {cert_name}") for path in (cert_dir, archive_dir): if path.exists(): @@ -568,8 +516,7 @@ def letsencrypt_delete(): for file in path.glob("*"): try: file.unlink() - if is_debug: - LOGGER.debug(f"Removed file: {file}") + debug_log(LOGGER, f"Removed file: {file}") except Exception as e: LOGGER.error(f"Failed to remove file {file}: " f"{e}") @@ -582,16 +529,14 @@ def letsencrypt_delete(): try: renewal_file.unlink() LOGGER.info(f"Removed renewal file {renewal_file}") - if is_debug: - LOGGER.debug(f"Renewal file removed: {renewal_file}") + debug_log(LOGGER, f"Renewal file removed: {renewal_file}") except Exception as e: LOGGER.error(f"Failed to remove renewal file " f"{renewal_file}: {e}") # Update database cache with modified certificate data try: - if is_debug: - LOGGER.debug("Updating database cache with modified data") + debug_log(LOGGER, "Updating database cache with modified data") dir_path = Path(LE_CACHE_DIR) file_name = f"folder:{dir_path.as_posix()}.tgz" @@ -616,8 +561,7 @@ def letsencrypt_delete(): "message": f"Failed to cache letsencrypt " f"dir: {err}"}) - if is_debug: - LOGGER.debug("Database cache updated successfully") + debug_log(LOGGER, "Database cache updated successfully") except Exception as e: error_msg = (f"Successfully deleted certificate {cert_name}, " @@ -639,15 +583,8 @@ def letsencrypt_delete(): @login_required def letsencrypt_static(filename): # Serve static files for the Let's Encrypt blueprint. - # - # Args: - # filename (str): Path to the static file to serve - # - # Returns: - # Response: Flask response object for the static file is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - LOGGER.debug(f"Serving static file: {filename}") + debug_log(LOGGER, f"Serving static file: {filename}") return letsencrypt.send_static_file(filename) \ No newline at end of file diff --git a/src/common/core/letsencrypt/ui/hooks.py b/src/common/core/letsencrypt/ui/hooks.py index f5d7337dfc..79e5ddee1d 100644 --- a/src/common/core/letsencrypt/ui/hooks.py +++ b/src/common/core/letsencrypt/ui/hooks.py @@ -19,25 +19,27 @@ } +def debug_log(logger, message): + # Log debug messages only when LOG_LEVEL environment variable is set to + # "debug" + if getenv("LOG_LEVEL") == "debug": + logger.debug(f"[DEBUG] {message}") + + def context_processor(): # Flask context processor to inject variables into templates. # # Provides template context data for the Let's Encrypt certificate # management interface. Injects column preferences and other UI # configuration data that templates need for proper rendering. - # - # Returns: - # dict: Dictionary containing template context variables including - # column preferences for DataTables and page visibility settings. - # Returns None for excluded paths that don't need context injection. logger = getLogger("UI") is_debug = getenv("LOG_LEVEL") == "debug" - if is_debug: - logger.debug("Context processor called") - logger.debug(f"Request path: {request.path}") - logger.debug(f"Request method: {request.method}") - logger.debug(f"Request endpoint: {getattr(request, 'endpoint', 'unknown')}") + debug_log(logger, "Context processor called") + debug_log(logger, f"Request path: {request.path}") + debug_log(logger, f"Request method: {request.method}") + debug_log(logger, + f"Request endpoint: {getattr(request, 'endpoint', 'unknown')}") # Skip context processing for system/auth pages that don't need it excluded_paths = [ @@ -49,35 +51,37 @@ def context_processor(): path_excluded = request.path.startswith(tuple(excluded_paths)) if path_excluded: - if is_debug: - logger.debug(f"Path {request.path} is excluded from context processing") - for excluded_path in excluded_paths: - if request.path.startswith(excluded_path): - logger.debug(f" Matched exclusion pattern: {excluded_path}") - break + debug_log(logger, + f"Path {request.path} is excluded from context processing") + for excluded_path in excluded_paths: + if request.path.startswith(excluded_path): + debug_log(logger, + f" Matched exclusion pattern: {excluded_path}") + break return None - if is_debug: - logger.debug(f"Processing context for path: {request.path}") - logger.debug(f"Column preferences to inject:") - column_names = { - "3": "Common Name", "4": "Issuer", "5": "Valid From", - "6": "Valid To", "7": "Preferred Profile", "8": "Challenge", - "9": "Key Type", "10": "OCSP Support", "11": "Serial Number", - "12": "Fingerprint", "13": "Version" - } - for col_id, visible in COLUMNS_PREFERENCES_DEFAULTS.items(): - col_name = column_names.get(col_id, f"Column {col_id}") - logger.debug(f" {col_name} (#{col_id}): {'visible' if visible else 'hidden'}") + debug_log(logger, f"Processing context for path: {request.path}") + debug_log(logger, "Column preferences to inject:") + column_names = { + "3": "Common Name", "4": "Issuer", "5": "Valid From", + "6": "Valid To", "7": "Preferred Profile", "8": "Challenge", + "9": "Key Type", "10": "OCSP Support", "11": "Serial Number", + "12": "Fingerprint", "13": "Version" + } + for col_id, visible in COLUMNS_PREFERENCES_DEFAULTS.items(): + col_name = column_names.get(col_id, f"Column {col_id}") + debug_log(logger, + f" {col_name} (#{col_id}): {'visible' if visible else 'hidden'}") # Prepare context data for templates data = { "columns_preferences_defaults_letsencrypt": COLUMNS_PREFERENCES_DEFAULTS } - if is_debug: - logger.debug(f"Context processor returning {len(data)} variables") - logger.debug(f"Context data keys: {list(data.keys())}") - logger.debug(f"Let's Encrypt preferences: {len(COLUMNS_PREFERENCES_DEFAULTS)} columns configured") + debug_log(logger, f"Context processor returning {len(data)} variables") + debug_log(logger, f"Context data keys: {list(data.keys())}") + debug_log(logger, + f"Let's Encrypt preferences: {len(COLUMNS_PREFERENCES_DEFAULTS)} " + f"columns configured") return data \ No newline at end of file From 9971336885b489e2306ef5e9e93f3e686f667d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Mon, 30 Jun 2025 17:53:22 +0200 Subject: [PATCH 12/15] fix copilot suggestion Injecting message via html() can expose the UI to XSS if the server response isn't fully sanitized. Consider using text() or a sanitization library to safely render error messages. --- .../ui/blueprints/static/js/main.js | 293 ++++++++---------- 1 file changed, 134 insertions(+), 159 deletions(-) diff --git a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js index f978202eab..46f4740520 100644 --- a/src/common/core/letsencrypt/ui/blueprints/static/js/main.js +++ b/src/common/core/letsencrypt/ui/blueprints/static/js/main.js @@ -1,3 +1,13 @@ +// Log debug messages only when LOG_LEVEL environment variable is set to +// "debug" +function debugLog(message) { + if (process.env.LOG_LEVEL === "debug") { + console.debug(`[DEBUG] ${message}`); + } +} + +// Main initialization function that waits for all dependencies to load +// before initializing the Let's Encrypt certificate management interface (async function waitForDependencies() { // Wait for jQuery while (typeof jQuery === "undefined") { @@ -18,12 +28,10 @@ const logLevel = process.env.LOG_LEVEL; const isDebug = logLevel === "debug"; - if (isDebug) { - console.debug("Initializing Let's Encrypt certificate management"); - console.debug("Log level:", logLevel); - console.debug("jQuery version:", $.fn.jquery); - console.debug("DataTables version:", $.fn.DataTable.version); - } + debugLog("Initializing Let's Encrypt certificate management"); + debugLog(`Log level: ${logLevel}`); + debugLog(`jQuery version: ${$.fn.jquery}`); + debugLog(`DataTables version: ${$.fn.DataTable.version}`); // Ensure i18next is loaded before using it const t = @@ -35,13 +43,11 @@ const isReadOnly = $("#is-read-only").val()?.trim() === "True"; const userReadOnly = $("#user-read-only").val()?.trim() === "True"; - if (isDebug) { - console.debug("Application state initialized:"); - console.debug("- Read-only mode:", isReadOnly); - console.debug("- User read-only:", userReadOnly); - console.debug("- Action lock:", actionLock); - console.debug("- CSRF token available:", !!$("#csrf_token").val()); - } + debugLog("Application state initialized:"); + debugLog(`- Read-only mode: ${isReadOnly}`); + debugLog(`- User read-only: ${userReadOnly}`); + debugLog(`- Action lock: ${actionLock}`); + debugLog(`- CSRF token available: ${!!$("#csrf_token").val()}`); const headers = [ { @@ -82,33 +88,30 @@ }, ]; - // Set up the delete confirmation modal for certificates + // Configure the delete confirmation modal for certificate deletion + // operations, supporting both single and multiple certificate + // deletion workflows const setupDeleteCertModal = (certs) => { - if (isDebug) { - console.debug("Setting up delete modal for certificates:", - certs); - console.debug("Modal setup - certificate count:", certs.length); - } + debugLog(`Setting up delete modal for certificates: ${ + certs.map(c => c.domain).join(", ")}`); + debugLog(`Modal setup - certificate count: ${certs.length}`); const $modalBody = $("#deleteCertContent"); $modalBody.empty(); if (certs.length === 1) { - if (isDebug) { - console.debug("Configuring modal for single certificate:", - certs[0].domain); - } + debugLog(`Configuring modal for single certificate: ${ + certs[0].domain}`); $modalBody.html( `

    You are about to delete the certificate for: ` + `${certs[0].domain}

    ` ); - $("#confirmDeleteCertBtn").data("cert-name", certs[0].domain); + $("#confirmDeleteCertBtn").data("cert-name", + certs[0].domain); } else { - if (isDebug) { - console.debug("Configuring modal for multiple certificates:", - certs.map(c => c.domain)); - } + debugLog(`Configuring modal for multiple certificates: ${ + certs.map(c => c.domain).join(", ")}`); const certList = certs .map((cert) => `
  • ${cert.domain}
    • `) @@ -123,34 +126,30 @@ ); } - if (isDebug) { - console.debug("Modal configuration completed"); - } + debugLog("Modal configuration completed"); }; - // Show error modal with title and message + // Display error modal with specified title and message for user + // feedback during certificate management operations const showErrorModal = (title, message) => { - if (isDebug) { - console.debug("Showing error modal:", title, message); - } + debugLog(`Showing error modal: ${title} - ${message}`); $("#errorModalLabel").text(title); - $("#errorModalContent").html(message); + $("#errorModalContent").text(message); const errorModal = new bootstrap.Modal( document.getElementById("errorModal") ); errorModal.show(); }; - // Handle delete button click events + // Handle delete button click events for certificate deletion + // confirmation modal $("#confirmDeleteCertBtn").on("click", function () { const certName = $(this).data("cert-name"); const certNames = $(this).data("cert-names"); - if (isDebug) { - console.debug("Delete button clicked:", - { certName, certNames }); - } + debugLog(`Delete button clicked: certName=${certName}, ` + + `certNames=${certNames}`); if (certName) { deleteCertificate(certName); @@ -172,23 +171,20 @@ $("#deleteCertModal").modal("hide"); }); - // Delete a single certificate with optional callback + // Delete a single certificate via AJAX request with optional callback + // for sequential deletion operations function deleteCertificate(certName, callback) { - if (isDebug) { - console.debug("Starting certificate deletion process:"); - console.debug("- Certificate name:", certName); - console.debug("- Has callback:", !!callback); - console.debug("- Request URL:", - `${window.location.pathname}/delete`); - } + debugLog("Starting certificate deletion process:"); + debugLog(`- Certificate name: ${certName}`); + debugLog(`- Has callback: ${!!callback}`); + debugLog(`- Request URL: ${ + window.location.pathname}/delete`); const requestData = { cert_name: certName }; const csrfToken = $("#csrf_token").val(); - if (isDebug) { - console.debug("Request payload:", requestData); - console.debug("CSRF token:", csrfToken ? "present" : "missing"); - } + debugLog(`Request payload: ${JSON.stringify(requestData)}`); + debugLog(`CSRF token: ${csrfToken ? "present" : "missing"}`); $.ajax({ url: `${window.location.pathname}/delete`, @@ -199,88 +195,72 @@ "X-CSRFToken": csrfToken, }, beforeSend: function(xhr) { - if (isDebug) { - console.debug("AJAX request starting for:", certName); - console.debug("Request headers:", xhr.getAllResponseHeaders()); - } + debugLog(`AJAX request starting for: ${certName}`); + debugLog(`Request headers: ${ + xhr.getAllResponseHeaders()}`); }, success: function (response) { - if (isDebug) { - console.debug("Delete response received:"); - console.debug("- Status:", response.status); - console.debug("- Message:", response.message); - console.debug("- Full response:", response); - } + debugLog("Delete response received:"); + debugLog(`- Status: ${response.status}`); + debugLog(`- Message: ${response.message}`); + debugLog(`- Full response: ${ + JSON.stringify(response)}`); if (response.status === "ok") { - if (isDebug) { - console.debug("Certificate deletion successful:", - certName); - } + debugLog(`Certificate deletion successful: ${ + certName}`); if (callback) { - if (isDebug) { - console.debug("Executing callback function"); - } + debugLog("Executing callback function"); callback(); } else { - if (isDebug) { - console.debug("Reloading DataTable data"); - } + debugLog("Reloading DataTable data"); $("#letsencrypt").DataTable().ajax.reload(); } } else { - if (isDebug) { - console.debug("Certificate deletion failed:", - response.message); - } + debugLog(`Certificate deletion failed: ${ + response.message}`); showErrorModal( "Certificate Deletion Error", - `

    Error deleting certificate ` + - `${certName}:

    ` + - `

    ${response.message || "Unknown error"}

    ` + `Error deleting certificate ${certName}: ${ + response.message || "Unknown error"}` ); if (callback) callback(); else $("#letsencrypt").DataTable().ajax.reload(); } }, error: function (xhr, status, error) { - if (isDebug) { - console.debug("AJAX error details:"); - console.debug("- XHR status:", xhr.status); - console.debug("- Status text:", status); - console.debug("- Error:", error); - console.debug("- Response text:", xhr.responseText); - console.debug("- Response JSON:", xhr.responseJSON); - } + debugLog("AJAX error details:"); + debugLog(`- XHR status: ${xhr.status}`); + debugLog(`- Status text: ${status}`); + debugLog(`- Error: ${error}`); + debugLog(`- Response text: ${xhr.responseText}`); + debugLog(`- Response JSON: ${ + JSON.stringify(xhr.responseJSON)}`); console.error("Error deleting certificate:", error, xhr); - let errorMessage = `

    Failed to delete certificate ` + - `${certName}:

    `; + let errorMessage = `Failed to delete certificate ` + + `${certName}: `; if (xhr.responseJSON && xhr.responseJSON.message) { - errorMessage += `

    ${xhr.responseJSON.message}

    `; + errorMessage += xhr.responseJSON.message; } else if (xhr.responseText) { try { const parsedError = JSON.parse(xhr.responseText); errorMessage += - `

    ${parsedError.message || error}

    `; + parsedError.message || error; } catch (e) { - if (isDebug) { - console.debug("Failed to parse error response:", - e); - } + debugLog(`Failed to parse error response: ${e}`); if (xhr.responseText.length < 200) { - errorMessage += `

    ${xhr.responseText}

    `; + errorMessage += xhr.responseText; } else { - errorMessage += `

    ${error || - "Unknown error"}

    `; + errorMessage += error || "Unknown error"; } } } else { - errorMessage += `

    ${error || "Unknown error"}

    `; + errorMessage += error || "Unknown error"; } showErrorModal("Certificate Deletion Failed", @@ -290,11 +270,9 @@ else $("#letsencrypt").DataTable().ajax.reload(); }, complete: function(xhr, status) { - if (isDebug) { - console.debug("AJAX request completed:"); - console.debug("- Final status:", status); - console.debug("- Certificate:", certName); - } + debugLog("AJAX request completed:"); + debugLog(`- Final status: ${status}`); + debugLog(`- Certificate: ${certName}`); } }); } @@ -337,13 +315,13 @@ }, }; - if (isDebug) { - console.debug("DataTable layout configuration:"); - console.debug("- Search panes columns:", layout.top1.searchPanes.columns); - console.debug("- Page length options:", layout.bottomStart.pageLength.menu); - console.debug("- Layout structure:", layout); - console.debug("- Headers count:", headers.length); - } + debugLog("DataTable layout configuration:"); + debugLog(`- Search panes columns: ${ + layout.top1.searchPanes.columns.join(", ")}`); + debugLog(`- Page length options: ${ + JSON.stringify(layout.bottomStart.pageLength.menu)}`); + debugLog(`- Layout structure: ${JSON.stringify(layout)}`); + debugLog(`- Headers count: ${headers.length}`); layout.topStart.buttons = [ { @@ -458,14 +436,13 @@ const sessionAutoRefresh = sessionStorage.getItem("letsencryptAutoRefresh"); - // Toggle auto-refresh functionality + // Toggle auto-refresh functionality for DataTable data with + // visual feedback and interval management function toggleAutoRefresh() { autoRefresh = !autoRefresh; sessionStorage.setItem("letsencryptAutoRefresh", autoRefresh); - if (isDebug) { - console.debug("Auto-refresh toggled:", autoRefresh); - } + debugLog(`Auto-refresh toggled: ${autoRefresh}`); if (autoRefresh) { $(".bx-loader") @@ -481,7 +458,8 @@ clearInterval(autoRefreshInterval); autoRefreshInterval = null; } else { - $("#letsencrypt").DataTable().ajax.reload(null, false); + $("#letsencrypt").DataTable().ajax.reload(null, + false); } }, 10000); } else { @@ -502,7 +480,8 @@ toggleAutoRefresh(); } - // Get currently selected certificates from DataTable + // Extract currently selected certificates from DataTable rows + // and return their domain information for bulk operations const getSelectedCertificates = () => { const certs = []; $("tr.selected").each(function () { @@ -511,14 +490,13 @@ certs.push({ domain: domain }); }); - if (isDebug) { - console.debug("Selected certificates:", certs); - } + debugLog(`Selected certificates: ${ + certs.map(c => c.domain).join(", ")}`); return certs; }; - // Custom DataTable button for auto-refresh + // Custom DataTable button for auto-refresh functionality $.fn.dataTable.ext.buttons.auto_refresh = { text: ( '' + @@ -530,7 +508,8 @@ }, }; - // Custom DataTable button for certificate deletion + // Custom DataTable button for certificate deletion with + // read-only mode checks and selection validation $.fn.dataTable.ext.buttons.delete_cert = { text: ( `` + @@ -568,7 +547,8 @@ }, }; - // Build column definitions for DataTable + // Build column definitions for DataTable configuration with + // responsive controls, selection, and search pane settings function buildColumnDefs() { return [ { @@ -639,7 +619,8 @@ ]; } - // Define the columns for the DataTable + // Define the columns for the DataTable with data mappings + // and display configurations for certificate information function buildColumns() { return [ { @@ -664,7 +645,8 @@ ]; } - // Manage header tooltips for DataTable columns + // Manage header tooltips for DataTable columns by applying + // Bootstrap tooltip attributes to table headers function updateHeaderTooltips(selector, headers) { $(selector) .find("th") @@ -684,7 +666,8 @@ $('[data-bs-toggle="tooltip"]').tooltip("dispose").tooltip(); } - // Initialize the DataTable with complete configuration + // Initialize the DataTable with complete configuration including + // server-side processing, AJAX data loading, and UI components const letsencrypt_config = { tableSelector: "#letsencrypt", tableName: "letsencrypt", @@ -706,28 +689,25 @@ url: `${window.location.pathname}/fetch`, type: "POST", data: function (d) { - if (isDebug) { - console.debug("DataTable AJAX request data:", d); - console.debug("Request parameters:"); - console.debug("- Draw:", d.draw); - console.debug("- Start:", d.start); - console.debug("- Length:", d.length); - console.debug("- Search value:", d.search?.value); - } + debugLog(`DataTable AJAX request data: ${ + JSON.stringify(d)}`); + debugLog("Request parameters:"); + debugLog(`- Draw: ${d.draw}`); + debugLog(`- Start: ${d.start}`); + debugLog(`- Length: ${d.length}`); + debugLog(`- Search value: ${d.search?.value}`); d.csrf_token = $("#csrf_token").val(); return d; }, error: function (jqXHR, textStatus, errorThrown) { - if (isDebug) { - console.debug("DataTable AJAX error details:"); - console.debug("- Status:", jqXHR.status); - console.debug("- Status text:", textStatus); - console.debug("- Error:", errorThrown); - console.debug("- Response text:", jqXHR.responseText); - console.debug("- Response headers:", - jqXHR.getAllResponseHeaders()); - } + debugLog("DataTable AJAX error details:"); + debugLog(`- Status: ${jqXHR.status}`); + debugLog(`- Status text: ${textStatus}`); + debugLog(`- Error: ${errorThrown}`); + debugLog(`- Response text: ${jqXHR.responseText}`); + debugLog(`- Response headers: ${ + jqXHR.getAllResponseHeaders()}`); console.error("DataTables AJAX error:", textStatus, errorThrown); @@ -742,21 +722,18 @@ $(".dataTables_processing").hide(); }, success: function(data, textStatus, jqXHR) { - if (isDebug) { - console.debug("DataTable AJAX success:"); - console.debug("- Records total:", data.recordsTotal); - console.debug("- Records filtered:", data.recordsFiltered); - console.debug("- Data length:", data.data?.length); - console.debug("- Draw number:", data.draw); - } + debugLog("DataTable AJAX success:"); + debugLog(`- Records total: ${data.recordsTotal}`); + debugLog(`- Records filtered: ${ + data.recordsFiltered}`); + debugLog(`- Data length: ${data.data?.length}`); + debugLog(`- Draw number: ${data.draw}`); } }, columns: buildColumns(), initComplete: function (settings, json) { - if (isDebug) { - console.debug("DataTable initialized with settings:", - settings); - } + debugLog(`DataTable initialized with settings: ${ + JSON.stringify(settings)}`); $("#letsencrypt_wrapper .btn-secondary") .removeClass("btn-secondary"); @@ -791,14 +768,12 @@ $(".tooltip").remove(); }); - // Toggle action button based on selection + // Toggle action button based on row selection state dt.on("select.dt deselect.dt", function () { const count = dt.rows({ selected: true }).count(); $(".action-button").toggleClass("disabled", count === 0); - if (isDebug) { - console.debug("Selection changed, count:", count); - } + debugLog(`Selection changed, count: ${count}`); }); }); })(); \ No newline at end of file From fc66e04c968907527575625137facc1cb9358133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:03:05 +0200 Subject: [PATCH 13/15] fixing comment lines --- src/common/core/letsencrypt/letsencrypt.lua | 28 +++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/common/core/letsencrypt/letsencrypt.lua b/src/common/core/letsencrypt/letsencrypt.lua index d7bb9ceb88..693223e536 100644 --- a/src/common/core/letsencrypt/letsencrypt.lua +++ b/src/common/core/letsencrypt/letsencrypt.lua @@ -28,7 +28,7 @@ local decode = cjson.decode local execute = os.execute local remove = os.remove --- Log debug messages only when LOG_LEVEL environment variable is set to +-- Log debug messages only when LOG_LEVEL environment variable is set to -- "debug" local function debug_log(logger, message) if os.getenv("LOG_LEVEL") == "debug" then @@ -37,12 +37,14 @@ local function debug_log(logger, message) end -- Initialize the letsencrypt plugin with the given context +-- @param ctx The context object containing plugin configuration function letsencrypt:initialize(ctx) - # Call parent initialize + -- Call parent initialize plugin.initialize(self, "letsencrypt", ctx) end -- Set the https_configured flag based on AUTO_LETS_ENCRYPT variable +-- Configures HTTPS settings for the plugin function letsencrypt:set() local https_configured = self.variables["AUTO_LETS_ENCRYPT"] if https_configured == "yes" then @@ -53,6 +55,8 @@ function letsencrypt:set() end -- Initialize SSL certificates and load them into the datastore +-- Handles both multisite and single-site configurations, processes wildcard +-- certificates, and loads certificate data from filesystem function letsencrypt:init() local ret_ok, ret_err = true, "success" local wildcard_servers = {} @@ -119,7 +123,8 @@ function letsencrypt:init() == "yes" then debug_log(self.logger, - "Using wildcard configuration for " .. server_name) + "Using wildcard configuration for " .. + server_name) for part in server_name:gmatch("%S+") do wildcard_servers[part] = true @@ -141,7 +146,7 @@ function letsencrypt:init() debug_log(self.logger, "Loading certificate files for " .. server_name) - # Load certificate + -- Load certificate local check check, data = read_files({ "/var/cache/bunkerweb/letsencrypt/etc/live/" .. @@ -265,6 +270,8 @@ function letsencrypt:init() end -- Handle SSL certificate selection based on SNI +-- Determines which certificate to use based on server name indication and +-- wildcard configuration function letsencrypt:ssl_certificate() debug_log(self.logger, "SSL certificate phase started") @@ -308,16 +315,20 @@ function letsencrypt:ssl_certificate() end -- Load certificate and private key data into the datastore +-- Parses PEM certificate and private key files and caches them in the +-- datastore for quick retrieval +-- @param data Table containing certificate and key file contents +-- @param server_name The server name to associate with the certificate function letsencrypt:load_data(data, server_name) debug_log(self.logger, "Loading certificate data for " .. server_name) - # Load certificate + -- Load certificate local cert_chain, err = parse_pem_cert(data[1]) if not cert_chain then return false, "error while parsing pem cert : " .. err end - # Load key + -- Load key local priv_key priv_key, err = parse_pem_priv_key(data[2]) if not priv_key then @@ -326,7 +337,7 @@ function letsencrypt:load_data(data, server_name) debug_log(self.logger, "Certificate and key parsed successfully") - # Cache data + -- Cache data for key in server_name:gmatch("%S+") do debug_log(self.logger, "Caching certificate data for " .. key) @@ -344,6 +355,7 @@ function letsencrypt:load_data(data, server_name) end -- Handle ACME challenge requests during certificate generation +-- Allows Let's Encrypt to access challenge files for domain validation function letsencrypt:access() debug_log(self.logger, "Access phase started") @@ -363,6 +375,8 @@ function letsencrypt:access() end -- Handle API requests for certificate challenge management +-- Provides endpoints for creating and removing ACME challenge validation +-- tokens during the certificate issuance process function letsencrypt:api() debug_log(self.logger, "API endpoint called") From 036e290492906fbbc91f4f2545f3127aceb8209c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Mon, 30 Jun 2025 18:04:47 +0200 Subject: [PATCH 14/15] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9619e05675..13a9966441 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ src/ui/templates src/ui/builder src/ui/client/builder/*.json src/ui/client/builder/*.txt +.vscode/settings.json From d2616cb1ba65be33f026b543c124346f7f2c90ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20K=C3=B6ckeis-Fresel?= <14275273+Michal-Koeckeis-Fresel@users.noreply.github.com> Date: Wed, 2 Jul 2025 07:30:25 +0200 Subject: [PATCH 15/15] fix Pylance errors using "strict" checks --- .../core/letsencrypt/jobs/certbot-new.py | 1403 ++++++++++------- 1 file changed, 799 insertions(+), 604 deletions(-) diff --git a/src/common/core/letsencrypt/jobs/certbot-new.py b/src/common/core/letsencrypt/jobs/certbot-new.py index d8c049f1b2..c2e76f1c74 100644 --- a/src/common/core/letsencrypt/jobs/certbot-new.py +++ b/src/common/core/letsencrypt/jobs/certbot-new.py @@ -17,7 +17,9 @@ from sys import exit as sys_exit, path as sys_path from time import sleep from traceback import format_exc -from typing import Dict, Literal, Optional, Type, Union +from typing import ( + Any, Dict, List, Literal, Optional, Set, Tuple, cast, Union +) for deps_path in [join(sep, "usr", "share", "bunkerweb", *paths) for paths in (("deps", "python"), ("utils",), ("db",))]: @@ -53,61 +55,69 @@ from jobs import Job # type: ignore from logger import setup_logger # type: ignore -LOGGER = setup_logger("LETS-ENCRYPT.new") -CERTBOT_BIN = join(sep, "usr", "share", "bunkerweb", "deps", "python", - "bin", "certbot") -DEPS_PATH = join(sep, "usr", "share", "bunkerweb", "deps", "python") +LOGGER: Any = setup_logger("LETS-ENCRYPT.new") +CERTBOT_BIN: str = join(sep, "usr", "share", "bunkerweb", "deps", "python", + "bin", "certbot") +DEPS_PATH: str = join(sep, "usr", "share", "bunkerweb", "deps", "python") -LOGGER_CERTBOT = setup_logger("LETS-ENCRYPT.new.certbot") -status = 0 +LOGGER_CERTBOT: Any = setup_logger("LETS-ENCRYPT.new.certbot") +status: int = 0 -PLUGIN_PATH = Path(sep, "usr", "share", "bunkerweb", "core", "letsencrypt") -JOBS_PATH = PLUGIN_PATH.joinpath("jobs") -CACHE_PATH = Path(sep, "var", "cache", "bunkerweb", "letsencrypt") -DATA_PATH = CACHE_PATH.joinpath("etc") -WORK_DIR = join(sep, "var", "lib", "bunkerweb", "letsencrypt") -LOGS_DIR = join(sep, "var", "log", "bunkerweb", "letsencrypt") +PLUGIN_PATH: Any = Path(sep, "usr", "share", "bunkerweb", "core", + "letsencrypt") +JOBS_PATH: Any = PLUGIN_PATH.joinpath("jobs") +CACHE_PATH: Any = Path(sep, "var", "cache", "bunkerweb", "letsencrypt") +DATA_PATH: Any = CACHE_PATH.joinpath("etc") +WORK_DIR: str = join(sep, "var", "lib", "bunkerweb", "letsencrypt") +LOGS_DIR: str = join(sep, "var", "log", "bunkerweb", "letsencrypt") -PSL_URL = "https://publicsuffix.org/list/public_suffix_list.dat" -PSL_STATIC_FILE = Path("public_suffix_list.dat") +PSL_URL: str = "https://publicsuffix.org/list/public_suffix_list.dat" +PSL_STATIC_FILE: Any = Path("public_suffix_list.dat") # ZeroSSL Configuration -ZEROSSL_ACME_SERVER = "https://acme.zerossl.com/v2/DV90" -ZEROSSL_STAGING_SERVER = "https://acme.zerossl.com/v2/DV90" -LETSENCRYPT_ACME_SERVER = "https://acme-v02.api.letsencrypt.org/directory" -LETSENCRYPT_STAGING_SERVER = ( +ZEROSSL_ACME_SERVER: str = "https://acme.zerossl.com/v2/DV90" +ZEROSSL_STAGING_SERVER: str = "https://acme.zerossl.com/v2/DV90" +LETSENCRYPT_ACME_SERVER: str = ( + "https://acme-v02.api.letsencrypt.org/directory" +) +LETSENCRYPT_STAGING_SERVER: str = ( "https://acme-staging-v02.api.letsencrypt.org/directory" ) -def debug_log(logger, message): +def debug_log(logger: Any, message: str) -> None: # Log debug messages only when LOG_LEVEL environment variable is set to # "debug" if getenv("LOG_LEVEL") == "debug": logger.debug(f"[DEBUG] {message}") -def load_public_suffix_list(job): +def load_public_suffix_list(job: Any) -> List[str]: # Load and cache the public suffix list for domain validation. # Fetches the PSL from the official source and caches it locally. # Returns cached version if available and fresh (less than 1 day old). debug_log(LOGGER, f"Loading public suffix list from cache or {PSL_URL}") debug_log(LOGGER, "Checking if cached PSL is available and fresh") - job_cache = job.get_cache(PSL_STATIC_FILE.name, with_info=True, - with_data=True) + job_cache: Union[Dict[str, Any], bool] = job.get_cache( + PSL_STATIC_FILE.name, with_info=True, with_data=True + ) if ( isinstance(job_cache, dict) - and job_cache.get("last_update") + and "last_update" in job_cache and job_cache["last_update"] < ( datetime.now().astimezone() - timedelta(days=1) ).timestamp() ): debug_log(LOGGER, "Using cached public suffix list") - cache_age_hours = ((datetime.now().astimezone().timestamp() - - job_cache['last_update']) / 3600) + cache_last_update: float = float(job_cache['last_update']) + cache_age_hours: float = ( + (datetime.now().astimezone().timestamp() - + cache_last_update) / 3600 + ) debug_log(LOGGER, f"Cache age: {cache_age_hours:.1f} hours") - return job_cache["data"].decode("utf-8").splitlines() + cache_data: bytes = cast(bytes, job_cache["data"]) + return cache_data.decode("utf-8").splitlines() try: debug_log(LOGGER, f"Downloading fresh PSL from {PSL_URL}") @@ -115,16 +125,20 @@ def load_public_suffix_list(job): resp = get(PSL_URL, timeout=5) resp.raise_for_status() - content = resp.text + content: str = resp.text - debug_log(LOGGER, f"Downloaded PSL successfully, {len(content)} bytes") - debug_log(LOGGER, f"PSL contains {len(content.splitlines())} lines") + debug_log(LOGGER, + f"Downloaded PSL successfully, {len(content)} bytes") + debug_log(LOGGER, + f"PSL contains {len(content.splitlines())} lines") + cached: Any + err: Any cached, err = JOB.cache_file(PSL_STATIC_FILE.name, - content.encode("utf-8")) + content.encode("utf-8")) if not cached: LOGGER.error(f"Error while saving public suffix list to cache: " - f"{err}") + f"{err}") else: debug_log(LOGGER, "PSL successfully cached for future use") @@ -133,11 +147,12 @@ def load_public_suffix_list(job): LOGGER.debug(format_exc()) LOGGER.error(f"Error while downloading public suffix list: {e}") - debug_log(LOGGER, "Download failed, checking for existing static file") + debug_log(LOGGER, + "Download failed, checking for existing static file") if PSL_STATIC_FILE.exists(): debug_log(LOGGER, - f"Using existing static PSL file: {PSL_STATIC_FILE}") + f"Using existing static PSL file: {PSL_STATIC_FILE}") with PSL_STATIC_FILE.open("r", encoding="utf-8") as f: return f.read().splitlines() @@ -145,17 +160,17 @@ def load_public_suffix_list(job): return [] -def parse_psl(psl_lines): +def parse_psl(psl_lines: List[str]) -> Dict[str, Set[str]]: # Parse PSL lines into rules and exceptions sets. # Processes the public suffix list format, handling comments, # exceptions (lines starting with !), and regular rules. debug_log(LOGGER, f"Parsing {len(psl_lines)} PSL lines") debug_log(LOGGER, "Processing rules, exceptions, and filtering comments") - rules = set() - exceptions = set() - comments_skipped = 0 - empty_lines_skipped = 0 + rules: Set[str] = set() + exceptions: Set[str] = set() + comments_skipped: int = 0 + empty_lines_skipped: int = 0 for line in psl_lines: line = line.strip() @@ -179,12 +194,12 @@ def parse_psl(psl_lines): return {"rules": rules, "exceptions": exceptions} -def is_domain_blacklisted(domain, psl): +def is_domain_blacklisted(domain: str, psl: Dict[str, Set[str]]) -> bool: # Check if domain is forbidden by PSL rules. # Validates whether a domain would be blacklisted according to the # Public Suffix List rules and exceptions. domain = domain.lower().strip(".") - labels = domain.split(".") + labels: List[str] = domain.split(".") debug_log(LOGGER, f"Checking domain {domain} against PSL rules") debug_log(LOGGER, f"Domain has {len(labels)} labels: {labels}") @@ -192,13 +207,13 @@ def is_domain_blacklisted(domain, psl): f"{len(psl['exceptions'])} exceptions") for i in range(len(labels)): - candidate = ".".join(str(label) for label in labels[i:]) + candidate: str = ".".join(str(label) for label in labels[i:]) debug_log(LOGGER, f"Checking candidate: {candidate}") if candidate in psl["exceptions"]: debug_log(LOGGER, f"Domain {domain} allowed by PSL exception " - f"{candidate}") + f"{candidate}") return False if candidate in psl["rules"]: @@ -207,52 +222,56 @@ def is_domain_blacklisted(domain, psl): if i == 0: debug_log(LOGGER, f"Domain {domain} blacklisted - exact PSL " - f"rule match") + f"rule match") return True if i == 0 and domain.startswith("*."): debug_log(LOGGER, f"Wildcard domain {domain} blacklisted - " - f"exact PSL rule match") + f"exact PSL rule match") return True if i == 0 or (i == 1 and labels[0] == "*"): debug_log(LOGGER, f"Domain {domain} blacklisted - PSL rule " - f"violation") + f"violation") return True if len(labels[i:]) == len(labels): debug_log(LOGGER, f"Domain {domain} blacklisted - full label " - f"match") + f"match") return True - wildcard_candidate = f"*.{candidate}" + wildcard_candidate: str = f"*.{candidate}" if wildcard_candidate in psl["rules"]: debug_log(LOGGER, f"Found PSL wildcard rule match: " - f"{wildcard_candidate}") + f"{wildcard_candidate}") if len(labels[i:]) == 2: debug_log(LOGGER, f"Domain {domain} blacklisted - wildcard " - f"PSL rule match") + f"PSL rule match") return True debug_log(LOGGER, f"Domain {domain} not blacklisted by PSL") return False -def get_certificate_authority_config(ca_provider, staging=False): +def get_certificate_authority_config( + ca_provider: str, + staging: bool = False +) -> Dict[str, str]: # Get ACME server configuration for the specified CA provider. # Returns the appropriate ACME server URL and name for the given # certificate authority and environment (staging/production). debug_log(LOGGER, f"Getting CA config for {ca_provider}, " f"staging={staging}") + config: Dict[str, str] if ca_provider.lower() == "zerossl": config = { "server": (ZEROSSL_STAGING_SERVER if staging - else ZEROSSL_ACME_SERVER), + else ZEROSSL_ACME_SERVER), "name": "ZeroSSL" } else: # Default to Let's Encrypt config = { "server": (LETSENCRYPT_STAGING_SERVER if staging - else LETSENCRYPT_ACME_SERVER), + else LETSENCRYPT_ACME_SERVER), "name": "Let's Encrypt" } @@ -261,7 +280,10 @@ def get_certificate_authority_config(ca_provider, staging=False): return config -def setup_zerossl_eab_credentials(email, api_key=None): +def setup_zerossl_eab_credentials( + email: str, + api_key: Optional[str] = None +) -> Tuple[Optional[str], Optional[str]]: # Setup External Account Binding (EAB) credentials for ZeroSSL. # Contacts the ZeroSSL API to obtain EAB credentials required for # ACME certificate issuance with ZeroSSL. @@ -270,7 +292,8 @@ def setup_zerossl_eab_credentials(email, api_key=None): if not api_key: LOGGER.error("❌ ZeroSSL API key not provided") LOGGER.warning( - "ZeroSSL API key not provided, attempting registration with email" + "ZeroSSL API key not provided, attempting registration with " + "email" ) return None, None @@ -298,26 +321,27 @@ def setup_zerossl_eab_credentials(email, api_key=None): if response.status_code == 200: response.raise_for_status() - eab_data = response.json() + eab_data: Dict[str, Any] = response.json() debug_log(LOGGER, f"ZeroSSL API response data: {eab_data}") LOGGER.info(f"ZeroSSL API response data: {eab_data}") # ZeroSSL typically returns eab_kid and eab_hmac_key directly if "eab_kid" in eab_data and "eab_hmac_key" in eab_data: - eab_kid = eab_data.get("eab_kid") - eab_hmac_key = eab_data.get("eab_hmac_key") + eab_kid: Optional[str] = eab_data.get("eab_kid") + eab_hmac_key: Optional[str] = eab_data.get("eab_hmac_key") LOGGER.info(f"✓ Successfully obtained EAB credentials from " - f"ZeroSSL") - kid_display = f"{eab_kid[:10]}..." if eab_kid else "None" - hmac_display = (f"{eab_hmac_key[:10]}..." if eab_hmac_key - else "None") + f"ZeroSSL") + kid_display: str = (f"{eab_kid[:10]}..." if eab_kid + else "None") + hmac_display: str = (f"{eab_hmac_key[:10]}..." + if eab_hmac_key else "None") LOGGER.info(f"EAB Kid: {kid_display}") LOGGER.info(f"EAB HMAC Key: {hmac_display}") return eab_kid, eab_hmac_key else: LOGGER.error(f"❌ Invalid ZeroSSL API response format: " - f"{eab_data}") + f"{eab_data}") return None, None else: # Try alternative endpoint if first one fails @@ -325,7 +349,7 @@ def setup_zerossl_eab_credentials(email, api_key=None): f"Primary endpoint failed with {response.status_code}, " "trying alternative" ) - response_text = response.text + response_text: str = response.text debug_log(LOGGER, f"Primary endpoint response: {response_text}") LOGGER.info(f"Primary endpoint response: {response_text}") @@ -340,16 +364,16 @@ def setup_zerossl_eab_credentials(email, api_key=None): timeout=30 ) - alt_status = response.status_code + alt_status: int = response.status_code debug_log(LOGGER, f"Alternative ZeroSSL API response status: " - f"{alt_status}") + f"{alt_status}") LOGGER.info(f"Alternative ZeroSSL API response status: " - f"{response.status_code}") + f"{response.status_code}") response.raise_for_status() eab_data = response.json() debug_log(LOGGER, f"Alternative ZeroSSL API response data: " - f"{eab_data}") + f"{eab_data}") LOGGER.info(f"Alternative ZeroSSL API response data: {eab_data}") if eab_data.get("success"): @@ -361,12 +385,13 @@ def setup_zerossl_eab_credentials(email, api_key=None): ) kid_display = f"{eab_kid[:10]}..." if eab_kid else "None" hmac_display = (f"{eab_hmac_key[:10]}..." if eab_hmac_key - else "None") + else "None") LOGGER.info(f"EAB Kid: {kid_display}") LOGGER.info(f"EAB HMAC Key: {hmac_display}") return eab_kid, eab_hmac_key else: - LOGGER.error(f"❌ ZeroSSL EAB registration failed: {eab_data}") + LOGGER.error(f"❌ ZeroSSL EAB registration failed: " + f"{eab_data}") return None, None except BaseException as e: @@ -385,14 +410,15 @@ def setup_zerossl_eab_credentials(email, api_key=None): return None, None -def get_caa_records(domain): +def get_caa_records(domain: str) -> Optional[List[Dict[str, str]]]: # Get CAA records for a domain using dig command. # Queries DNS CAA records to check certificate authority authorization. # Returns None if dig command is not available. # Check if dig command is available if not which("dig"): - debug_log(LOGGER, "dig command not available for CAA record checking") + debug_log(LOGGER, + "dig command not available for CAA record checking") LOGGER.info("dig command not available for CAA record checking") return None @@ -415,8 +441,8 @@ def get_caa_records(domain): if result.returncode == 0 and result.stdout.strip(): LOGGER.info(f"Found CAA records for domain {domain}") - caa_records = [] - raw_lines = result.stdout.strip().split('\n') + caa_records: List[Dict[str, str]] = [] + raw_lines: List[str] = result.stdout.strip().split('\n') debug_log(LOGGER, f"Processing {len(raw_lines)} CAA record lines") @@ -427,30 +453,32 @@ def get_caa_records(domain): # CAA record format: flags tag "value" # Example: 0 issue "letsencrypt.org" - parts = line.split(' ', 2) + parts: List[str] = line.split(' ', 2) if len(parts) >= 3: - flags = parts[0] - tag = parts[1] - value = parts[2].strip('"') + flags: str = parts[0] + tag: str = parts[1] + value: str = parts[2].strip('"') caa_records.append({ 'flags': flags, 'tag': tag, 'value': value }) - debug_log(LOGGER, f"Parsed CAA record: flags={flags}, " - f"tag={tag}, value={value}") + debug_log(LOGGER, + f"Parsed CAA record: flags={flags}, " + f"tag={tag}, value={value}") - record_count = len(caa_records) - debug_log(LOGGER, f"Successfully parsed {record_count} CAA records " - f"for domain {domain}") + record_count: int = len(caa_records) + debug_log(LOGGER, + f"Successfully parsed {record_count} CAA records " + f"for domain {domain}") LOGGER.info(f"Parsed {len(caa_records)} CAA records for domain " - f"{domain}") + f"{domain}") return caa_records else: debug_log(LOGGER, - f"No CAA records found for domain {domain} " - f"(dig return code: {result.returncode})") + f"No CAA records found for domain {domain} " + f"(dig return code: {result.returncode})") LOGGER.info( f"No CAA records found for domain {domain} " f"(dig return code: {result.returncode})" @@ -463,13 +491,17 @@ def get_caa_records(domain): return None -def check_caa_authorization(domain, ca_provider, is_wildcard=False): +def check_caa_authorization( + domain: str, + ca_provider: str, + is_wildcard: bool = False +) -> bool: # Check if the CA provider is authorized by CAA records. # Validates whether the certificate authority is permitted to issue # certificates for the domain according to CAA DNS records. debug_log(LOGGER, - f"Checking CAA authorization for domain: {domain}, " - f"CA: {ca_provider}, wildcard: {is_wildcard}") + f"Checking CAA authorization for domain: {domain}, " + f"CA: {ca_provider}, wildcard: {is_wildcard}") LOGGER.info( f"Checking CAA authorization for domain: {domain}, " @@ -477,12 +509,14 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): ) # Map CA providers to their CAA identifiers - ca_identifiers = { + ca_identifiers: Dict[str, List[str]] = { "letsencrypt": ["letsencrypt.org"], "zerossl": ["sectigo.com", "zerossl.com"] # ZeroSSL uses Sectigo } - allowed_identifiers = ca_identifiers.get(ca_provider.lower(), []) + allowed_identifiers: List[str] = ca_identifiers.get( + ca_provider.lower(), [] + ) if not allowed_identifiers: LOGGER.warning(f"Unknown CA provider for CAA check: {ca_provider}") debug_log(LOGGER, "Returning True for unknown CA provider " @@ -493,8 +527,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"{allowed_identifiers}") # Check CAA records for the domain and parent domains - check_domain = domain.lstrip("*.") - domain_parts = check_domain.split(".") + check_domain: str = domain.lstrip("*.") + domain_parts: List[str] = check_domain.split(".") debug_log(LOGGER, f"Will check CAA records for domain chain: " f"{check_domain}") @@ -502,16 +536,20 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): LOGGER.info(f"Will check CAA records for domain chain: {check_domain}") for i in range(len(domain_parts)): - current_domain = ".".join(str(part) for part in domain_parts[i:]) + current_domain: str = ".".join( + str(part) for part in domain_parts[i:] + ) debug_log(LOGGER, f"Checking CAA records for: {current_domain}") LOGGER.info(f"Checking CAA records for: {current_domain}") - caa_records = get_caa_records(current_domain) + caa_records: Optional[List[Dict[str, str]]] = get_caa_records( + current_domain + ) if caa_records is None: # dig not available, skip CAA check LOGGER.info("CAA record checking skipped (dig command not " - "available)") + "available)") debug_log(LOGGER, "Returning True due to unavailable dig command") return True @@ -519,8 +557,8 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): LOGGER.info(f"Found CAA records for {current_domain}") # Check relevant CAA records - issue_records = [] - issuewild_records = [] + issue_records: List[str] = [] + issuewild_records: List[str] = [] for record in caa_records: if record['tag'] == 'issue': @@ -531,38 +569,40 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): # Log found records if issue_records: debug_log(LOGGER, f"CAA issue records: " - f"{', '.join(str(record) for record in issue_records)}") + f"{', '.join(str(record) for record in issue_records)}") LOGGER.info(f"CAA issue records: " - f"{', '.join(str(record) for record in issue_records)}") + f"{', '.join(str(record) for record in issue_records)}") if issuewild_records: debug_log(LOGGER, f"CAA issuewild records: " - f"{', '.join(str(record) for record in issuewild_records)}") + f"{', '.join(str(record) for record in issuewild_records)}") LOGGER.info(f"CAA issuewild records: " - f"{', '.join(str(record) for record in issuewild_records)}") + f"{', '.join(str(record) for record in issuewild_records)}") # Check authorization based on certificate type + check_records: List[str] + record_type: str if is_wildcard: # For wildcard certificates, check issuewild first, # then fall back to issue check_records = (issuewild_records if issuewild_records - else issue_records) + else issue_records) record_type = ("issuewild" if issuewild_records - else "issue") + else "issue") else: # For regular certificates, check issue records check_records = issue_records record_type = "issue" debug_log(LOGGER, f"Using CAA {record_type} records for " - f"authorization check") + f"authorization check") debug_log(LOGGER, f"Records to check: {check_records}") LOGGER.info(f"Using CAA {record_type} records for authorization " - f"check") + f"check") if not check_records: debug_log(LOGGER, - f"No relevant CAA {record_type} records found for " - f"{current_domain}") + f"No relevant CAA {record_type} records found for " + f"{current_domain}") LOGGER.info( f"No relevant CAA {record_type} records found for " f"{current_domain}" @@ -570,29 +610,33 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): continue # Check if any of our CA identifiers are authorized - authorized = False + authorized: bool = False - identifier_list = ', '.join(str(id) for id in allowed_identifiers) + identifier_list: str = ', '.join( + str(id) for id in allowed_identifiers + ) debug_log(LOGGER, - f"Checking authorization for CA identifiers: " - f"{identifier_list}") + f"Checking authorization for CA identifiers: " + f"{identifier_list}") LOGGER.info( f"Checking authorization for CA identifiers: " f"{identifier_list}" ) for identifier in allowed_identifiers: for record in check_records: - debug_log(LOGGER, f"Comparing identifier '{identifier}' " - f"with record '{record}'") + debug_log(LOGGER, + f"Comparing identifier '{identifier}' " + f"with record '{record}'") # Handle explicit deny (empty value or semicolon) + if record == ";" or record.strip() == "": LOGGER.warning( f"CAA {record_type} record explicitly denies " f"all CAs" ) debug_log(LOGGER, "Found explicit deny record - " - "authorization failed") + "authorization failed") return False # Check for CA authorization @@ -602,8 +646,9 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"✓ CA {ca_provider} ({identifier}) authorized " f"by CAA {record_type} record" ) - debug_log(LOGGER, f"Authorization found: {identifier} " - f"in {record}") + debug_log(LOGGER, + f"Authorization found: {identifier} " + f"in {record}") break if authorized: break @@ -613,8 +658,12 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"❌ CA {ca_provider} is NOT authorized by " f"CAA {record_type} records" ) - allowed_list = ', '.join(str(record) for record in check_records) - identifier_list = ', '.join(str(id) for id in allowed_identifiers) + allowed_list: str = ', '.join( + str(record) for record in check_records + ) + identifier_list = ', '.join( + str(id) for id in allowed_identifiers + ) LOGGER.error( f"Domain {current_domain} CAA {record_type} allows: " f"{allowed_list}" @@ -623,14 +672,15 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): f"But {ca_provider} uses: {identifier_list}" ) debug_log(LOGGER, "CAA authorization failed - no matching " - "identifiers") + "identifiers") return False # If we found CAA records and we're authorized, we can stop # checking parent domains LOGGER.info(f"✓ CAA authorization successful for {domain}") - debug_log(LOGGER, "CAA authorization successful - stopping parent " - "domain checks") + debug_log(LOGGER, + "CAA authorization successful - stopping parent " + "domain checks") return True # No CAA records found in the entire chain @@ -643,49 +693,52 @@ def check_caa_authorization(domain, ca_provider, is_wildcard=False): return True -def validate_domains_for_http_challenge(domains_list, - ca_provider="letsencrypt", - is_wildcard=False): - # Validate that all domains have valid A/AAAA records and CAA authorization - # for HTTP challenge. +def validate_domains_for_http_challenge( + domains_list: List[str], + ca_provider: str = "letsencrypt", + is_wildcard: bool = False +) -> bool: + # Validate that all domains have valid A/AAAA records and CAA + # authorization for HTTP challenge. # Checks DNS resolution and certificate authority authorization for each # domain in the list to ensure HTTP challenge will succeed. - domain_count = len(domains_list) - domain_list = ', '.join(str(domain) for domain in domains_list) + domain_count: int = len(domains_list) + domain_list: str = ', '.join(str(domain) for domain in domains_list) + debug_log(LOGGER, + f"Validating {domain_count} domains for HTTP challenge: " + f"{domain_list}") debug_log(LOGGER, - f"Validating {domain_count} domains for HTTP challenge: " - f"{domain_list}") - debug_log(LOGGER, f"CA provider: {ca_provider}, wildcard: {is_wildcard}") + f"CA provider: {ca_provider}, wildcard: {is_wildcard}") LOGGER.info( f"Validating {len(domains_list)} domains for HTTP challenge: " f"{domain_list}" ) - invalid_domains = [] - caa_blocked_domains = [] + invalid_domains: List[str] = [] + caa_blocked_domains: List[str] = [] # Check if CAA validation should be skipped - skip_caa_check = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" + skip_caa_check: bool = getenv("ACME_SKIP_CAA_CHECK", "no") == "yes" - caa_status = 'skipped' if skip_caa_check else 'performed' + caa_status: str = 'skipped' if skip_caa_check else 'performed' debug_log(LOGGER, f"CAA check will be {caa_status}") # Get external IPs once for all domain checks - external_ips = get_external_ip() + external_ips: Optional[Dict[str, Optional[str]]] = get_external_ip() if external_ips: if external_ips.get("ipv4"): LOGGER.info(f"Server external IPv4 address: " - f"{external_ips['ipv4']}") + f"{external_ips['ipv4']}") if external_ips.get("ipv6"): LOGGER.info(f"Server external IPv6 address: " - f"{external_ips['ipv6']}") + f"{external_ips['ipv6']}") else: LOGGER.warning( "Could not determine server external IP - skipping IP match " "validation" ) - validation_passed = 0 - validation_failed = 0 + validation_passed: int = 0 + validation_failed: int = 0 for domain in domains_list: debug_log(LOGGER, f"Validating domain: {domain}") @@ -705,9 +758,9 @@ def validate_domains_for_http_challenge(domains_list, debug_log(LOGGER, f"CAA authorization failed for {domain}") else: debug_log(LOGGER, f"CAA check skipped for {domain} " - f"(ACME_SKIP_CAA_CHECK=yes)") + f"(ACME_SKIP_CAA_CHECK=yes)") LOGGER.info(f"CAA check skipped for {domain} " - f"(ACME_SKIP_CAA_CHECK=yes)") + f"(ACME_SKIP_CAA_CHECK=yes)") validation_passed += 1 debug_log(LOGGER, f"Validation passed for {domain}") @@ -717,7 +770,9 @@ def validate_domains_for_http_challenge(domains_list, # Report results if invalid_domains: - invalid_list = ', '.join(str(domain) for domain in invalid_domains) + invalid_list: str = ', '.join( + str(domain) for domain in invalid_domains + ) LOGGER.error( f"The following domains do not have valid A/AAAA records and " f"cannot be used for HTTP challenge: {invalid_list}" @@ -729,7 +784,9 @@ def validate_domains_for_http_challenge(domains_list, return False if caa_blocked_domains: - blocked_list = ', '.join(str(domain) for domain in caa_blocked_domains) + blocked_list: str = ', '.join( + str(domain) for domain in caa_blocked_domains + ) LOGGER.error( f"The following domains have CAA records that block " f"{ca_provider}: {blocked_list}" @@ -739,10 +796,10 @@ def validate_domains_for_http_challenge(domains_list, "authority or use a different CA" ) LOGGER.info("You can skip CAA checking by setting " - "ACME_SKIP_CAA_CHECK=yes") + "ACME_SKIP_CAA_CHECK=yes") return False - valid_list = ', '.join(str(domain) for domain in domains_list) + valid_list: str = ', '.join(str(domain) for domain in domains_list) LOGGER.info( f"All domains have valid DNS records and CAA authorization for " f"HTTP challenge: {valid_list}" @@ -750,21 +807,22 @@ def validate_domains_for_http_challenge(domains_list, return True -def get_external_ip(): - # Get the external/public IP addresses of this server (both IPv4 and IPv6). +def get_external_ip() -> Optional[Dict[str, Optional[str]]]: + # Get the external/public IP addresses of this server (both IPv4 + # and IPv6). # Queries multiple external services to determine the server's public # IP addresses for DNS validation purposes. debug_log(LOGGER, "Getting external IP addresses for server") LOGGER.info("Getting external IP addresses for server") - ipv4_services = [ + ipv4_services: List[str] = [ "https://ipv4.icanhazip.com", "https://api.ipify.org", "https://checkip.amazonaws.com", "https://ipv4.jsonip.com" ] - ipv6_services = [ + ipv6_services: List[str] = [ "https://ipv6.icanhazip.com", "https://api6.ipify.org", "https://ipv6.jsonip.com" @@ -779,43 +837,46 @@ def get_external_ip(): for i, service in enumerate(ipv4_services): try: - service_num = f"{i+1}/{len(ipv4_services)}" - debug_log(LOGGER, f"Trying IPv4 service {service_num}: {service}") + service_num: str = f"{i+1}/{len(ipv4_services)}" + debug_log(LOGGER, + f"Trying IPv4 service {service_num}: {service}") if "jsonip.com" in service: # This service returns JSON format response = get(service, timeout=5) response.raise_for_status() - data = response.json() - ip = data.get("ip", "").strip() + json_data: Dict[str, Any] = response.json() + ip_str: str = json_data.get("ip", "").strip() else: # These services return plain text IP response = get(service, timeout=5) response.raise_for_status() - ip = response.text.strip() + ip_str = response.text.strip() - debug_log(LOGGER, f"Service returned: {ip}") + debug_log(LOGGER, f"Service returned: {ip_str}") # Basic IPv4 validation - if ip and "." in ip and len(ip.split(".")) == 4: + if ip_str and "." in ip_str and len(ip_str.split(".")) == 4: try: # Validate it's a proper IPv4 address - getaddrinfo(ip, None, AF_INET) + getaddrinfo(ip_str, None, AF_INET) # Type-safe assignment - ipv4_addr: str = str(ip) + ipv4_addr: str = str(ip_str) external_ips["ipv4"] = ipv4_addr - debug_log(LOGGER, f"Successfully obtained external IPv4 " - f"address: {ipv4_addr}") + debug_log(LOGGER, + f"Successfully obtained external IPv4 " + f"address: {ipv4_addr}") LOGGER.info(f"Successfully obtained external IPv4 " - f"address: {ipv4_addr}") + f"address: {ipv4_addr}") break except gaierror: - debug_log(LOGGER, f"Invalid IPv4 address returned: {ip}") + debug_log(LOGGER, + f"Invalid IPv4 address returned: {ip_str}") continue except BaseException as e: - debug_log(LOGGER, f"Failed to get IPv4 address from {service}: " - f"{e}") + debug_log(LOGGER, + f"Failed to get IPv4 address from {service}: {e}") LOGGER.info(f"Failed to get IPv4 address from {service}: {e}") continue @@ -827,40 +888,43 @@ def get_external_ip(): for i, service in enumerate(ipv6_services): try: service_num = f"{i+1}/{len(ipv6_services)}" - debug_log(LOGGER, f"Trying IPv6 service {service_num}: {service}") + debug_log(LOGGER, + f"Trying IPv6 service {service_num}: {service}") if "jsonip.com" in service: response = get(service, timeout=5) response.raise_for_status() - data = response.json() - ip = data.get("ip", "").strip() + json_data: Dict[str, Any] = response.json() + ip_str = json_data.get("ip", "").strip() else: response = get(service, timeout=5) response.raise_for_status() - ip = response.text.strip() + ip_str = response.text.strip() - debug_log(LOGGER, f"Service returned: {ip}") + debug_log(LOGGER, f"Service returned: {ip_str}") # Basic IPv6 validation - if ip and ":" in ip: + if ip_str and ":" in ip_str: try: # Validate it's a proper IPv6 address - getaddrinfo(ip, None, AF_INET6) + getaddrinfo(ip_str, None, AF_INET6) # Type-safe assignment - ipv6_addr: str = str(ip) + ipv6_addr: str = str(ip_str) external_ips["ipv6"] = ipv6_addr - debug_log(LOGGER, f"Successfully obtained external IPv6 " - f"address: {ipv6_addr}") + debug_log(LOGGER, + f"Successfully obtained external IPv6 " + f"address: {ipv6_addr}") LOGGER.info(f"Successfully obtained external IPv6 " - f"address: {ipv6_addr}") + f"address: {ipv6_addr}") break except gaierror: - debug_log(LOGGER, f"Invalid IPv6 address returned: {ip}") + debug_log(LOGGER, + f"Invalid IPv6 address returned: {ip_str}") continue except BaseException as e: - debug_log(LOGGER, f"Failed to get IPv6 address from {service}: " - f"{e}") + debug_log(LOGGER, + f"Failed to get IPv6 address from {service}: {e}") LOGGER.info(f"Failed to get IPv6 address from {service}: {e}") continue @@ -872,8 +936,8 @@ def get_external_ip(): debug_log(LOGGER, "All external IP services failed") return None - ipv4_status = external_ips['ipv4'] or 'not found' - ipv6_status = external_ips['ipv6'] or 'not found' + ipv4_status: str = external_ips['ipv4'] or 'not found' + ipv6_status: str = external_ips['ipv6'] or 'not found' LOGGER.info( f"External IP detection completed - " f"IPv4: {ipv4_status}, IPv6: {ipv6_status}" @@ -881,26 +945,33 @@ def get_external_ip(): return external_ips -def check_domain_a_record(domain, external_ips=None): +def check_domain_a_record( + domain: str, + external_ips: Optional[Dict[str, Optional[str]]] = None +) -> bool: # Check if domain has valid A/AAAA records for HTTP challenge. # Validates DNS resolution and optionally checks if the domain's # IP addresses match the server's external IPs. debug_log(LOGGER, f"Checking DNS A/AAAA records for domain: {domain}") LOGGER.info(f"Checking DNS A/AAAA records for domain: {domain}") + + # Remove wildcard prefix if present + check_domain: str = domain.lstrip("*.") + try: - # Remove wildcard prefix if present - check_domain = domain.lstrip("*.") debug_log(LOGGER, f"Checking domain after wildcard removal: " f"{check_domain}") # Attempt to resolve the domain to IP addresses - result = getaddrinfo(check_domain, None) + result: List[Tuple[Any, ...]] = getaddrinfo(check_domain, None) if result: - ipv4_addresses = [addr[4][0] for addr in result - if addr[0] == AF_INET] - ipv6_addresses = [addr[4][0] for addr in result - if addr[0] == AF_INET6] + ipv4_addresses: List[str] = [ + addr[4][0] for addr in result if addr[0] == AF_INET + ] + ipv6_addresses: List[str] = [ + addr[4][0] for addr in result if addr[0] == AF_INET6 + ] debug_log(LOGGER, "DNS resolution results:") debug_log(LOGGER, f" IPv4 addresses: {ipv4_addresses}") @@ -908,25 +979,29 @@ def check_domain_a_record(domain, external_ips=None): if not ipv4_addresses and not ipv6_addresses: LOGGER.warning(f"Domain {check_domain} has no A or AAAA " - f"records") + f"records") debug_log(LOGGER, "No valid IP addresses found in DNS " - "resolution") + "resolution") return False # Log found addresses if ipv4_addresses: - ipv4_display = ', '.join(str(addr) for addr in ipv4_addresses[:3]) + ipv4_display: str = ', '.join( + str(addr) for addr in ipv4_addresses[:3] + ) debug_log(LOGGER, - f"Domain {check_domain} IPv4 A records: " - f"{ipv4_display}") + f"Domain {check_domain} IPv4 A records: " + f"{ipv4_display}") LOGGER.info( f"Domain {check_domain} IPv4 A records: {ipv4_display}" ) if ipv6_addresses: - ipv6_display = ', '.join(str(addr) for addr in ipv6_addresses[:3]) + ipv6_display: str = ', '.join( + str(addr) for addr in ipv6_addresses[:3] + ) debug_log(LOGGER, - f"Domain {check_domain} IPv6 AAAA records: " - f"{ipv6_display}") + f"Domain {check_domain} IPv6 AAAA records: " + f"{ipv6_display}") LOGGER.info( f"Domain {check_domain} IPv6 AAAA records: " f"{ipv6_display}" @@ -934,16 +1009,17 @@ def check_domain_a_record(domain, external_ips=None): # Check if any record matches the external IPs if external_ips: - ipv4_match = False - ipv6_match = False + ipv4_match: bool = False + ipv6_match: bool = False - debug_log(LOGGER, "Checking IP address matches with server " - "external IPs") + debug_log(LOGGER, + "Checking IP address matches with server " + "external IPs") # Check IPv4 match if external_ips.get("ipv4") and ipv4_addresses: if external_ips["ipv4"] in ipv4_addresses: - external_ipv4 = external_ips['ipv4'] + external_ipv4: str = external_ips['ipv4'] LOGGER.info( f"✓ Domain {check_domain} IPv4 A record matches " f"server external IP ({external_ipv4})" @@ -951,18 +1027,20 @@ def check_domain_a_record(domain, external_ips=None): ipv4_match = True else: LOGGER.warning( - f"⚠ Domain {check_domain} IPv4 A record does not " - "match server external IP" + f"⚠ Domain {check_domain} IPv4 A record does " + "not match server external IP" + ) + ipv4_list: str = ', '.join( + str(addr) for addr in ipv4_addresses ) - ipv4_list = ', '.join(str(addr) for addr in ipv4_addresses) LOGGER.warning(f" Domain IPv4: {ipv4_list}") LOGGER.warning(f" Server IPv4: " - f"{external_ips['ipv4']}") + f"{external_ips['ipv4']}") # Check IPv6 match if external_ips.get("ipv6") and ipv6_addresses: if external_ips["ipv6"] in ipv6_addresses: - external_ipv6 = external_ips['ipv6'] + external_ipv6: str = external_ips['ipv6'] LOGGER.info( f"✓ Domain {check_domain} IPv6 AAAA record " f"matches server external IP ({external_ipv6})" @@ -973,20 +1051,23 @@ def check_domain_a_record(domain, external_ips=None): f"⚠ Domain {check_domain} IPv6 AAAA record does " "not match server external IP" ) - ipv6_list = ', '.join(str(addr) for addr in ipv6_addresses) + ipv6_list: str = ', '.join( + str(addr) for addr in ipv6_addresses + ) LOGGER.warning(f" Domain IPv6: {ipv6_list}") LOGGER.warning(f" Server IPv6: " - f"{external_ips['ipv6']}") + f"{external_ips['ipv6']}") # Determine if we have any matching records - has_any_match = ipv4_match or ipv6_match - has_external_ip = (external_ips.get("ipv4") or - external_ips.get("ipv6")) + has_any_match: bool = ipv4_match or ipv6_match + has_external_ip: bool = bool( + external_ips.get("ipv4") or external_ips.get("ipv6") + ) debug_log(LOGGER, f"IP match results: IPv4={ipv4_match}, " - f"IPv6={ipv6_match}") + f"IPv6={ipv6_match}") debug_log(LOGGER, f"Has external IP: {has_external_ip}, " - f"Has match: {has_any_match}") + f"Has match: {has_any_match}") if has_external_ip and not has_any_match: LOGGER.warning( @@ -999,15 +1080,16 @@ def check_domain_a_record(domain, external_ips=None): ) # Check if we should treat this as an error - strict_ip_check = (getenv("ACME_HTTP_STRICT_IP_CHECK", - "no") == "yes") + strict_ip_check: bool = ( + getenv("ACME_HTTP_STRICT_IP_CHECK", "no") == "yes" + ) if strict_ip_check: LOGGER.error( - f"Strict IP check enabled - rejecting certificate " - f"request for {check_domain}" + f"Strict IP check enabled - rejecting " + f"certificate request for {check_domain}" ) debug_log(LOGGER, "Strict IP check failed - " - "returning False") + "returning False") return False LOGGER.info(f"✓ Domain {check_domain} DNS validation passed") @@ -1015,10 +1097,10 @@ def check_domain_a_record(domain, external_ips=None): return True else: debug_log(LOGGER, - f"Domain {check_domain} validation failed - no DNS " - f"resolution") + f"Domain {check_domain} validation failed - no DNS " + f"resolution") LOGGER.info(f"Domain {check_domain} validation failed - no DNS " - f"resolution") + f"resolution") LOGGER.warning(f"Domain {check_domain} does not resolve") return False @@ -1026,14 +1108,14 @@ def check_domain_a_record(domain, external_ips=None): debug_log(LOGGER, f"Domain {check_domain} DNS resolution failed " f"(gaierror): {e}") LOGGER.info(f"Domain {check_domain} DNS resolution failed " - f"(gaierror): {e}") + f"(gaierror): {e}") LOGGER.warning(f"DNS resolution failed for domain {check_domain}: " - f"{e}") + f"{e}") return False except BaseException as e: LOGGER.info(format_exc()) LOGGER.error(f"Error checking DNS records for domain " - f"{check_domain}: {e}") + f"{check_domain}: {e}") debug_log(LOGGER, "DNS check failed with unexpected exception") return False @@ -1043,7 +1125,7 @@ def certbot_new_with_retry( domains: str, email: str, provider: Optional[str] = None, - credentials_path: Optional[Union[str, Path]] = None, + credentials_path: Optional[Any] = None, propagation: str = "default", profile: str = "classic", staging: bool = False, @@ -1061,24 +1143,26 @@ def certbot_new_with_retry( debug_log(LOGGER, f"Max retries: {max_retries}, CA: {ca_provider}") debug_log(LOGGER, f"Challenge: {challenge_type}, Provider: {provider}") - attempt = 1 + attempt: int = 1 + result: int = 1 # Initialize result while attempt <= max_retries + 1: if attempt > 1: LOGGER.warning( f"Certificate generation failed, retrying... " f"(attempt {attempt}/{max_retries + 1})" ) - wait_time = min(30 * (2 ** (attempt - 2)), 300) + wait_time: int = min(30 * (2 ** (attempt - 2)), 300) - debug_log(LOGGER, f"Waiting {wait_time} seconds before retry...") + debug_log(LOGGER, + f"Waiting {wait_time} seconds before retry...") debug_log(LOGGER, f"Exponential backoff: base=30s, " - f"attempt={attempt}") + f"attempt={attempt}") LOGGER.info(f"Waiting {wait_time} seconds before retry...") sleep(wait_time) debug_log(LOGGER, f"Executing certbot attempt {attempt}") - result = certbot_new( + certbot_result: int = certbot_new( challenge_type, domains, email, @@ -1094,18 +1178,20 @@ def certbot_new_with_retry( server_name, ) - if result == 0: + if certbot_result == 0: if attempt > 1: LOGGER.info(f"Certificate generation succeeded on attempt " - f"{attempt}") + f"{attempt}") debug_log(LOGGER, "Certbot completed successfully") - return result + return certbot_result if attempt >= max_retries + 1: LOGGER.error(f"Certificate generation failed after " - f"{max_retries + 1} attempts") + f"{max_retries + 1} attempts") debug_log(LOGGER, "Maximum retries reached - giving up") - return result + return certbot_result + + result = certbot_result # Update the outer result debug_log(LOGGER, f"Attempt {attempt} failed, will retry") attempt += 1 @@ -1118,7 +1204,7 @@ def certbot_new( domains: str, email: str, provider: Optional[str] = None, - credentials_path: Optional[Union[str, Path]] = None, + credentials_path: Optional[Any] = None, propagation: str = "default", profile: str = "classic", staging: bool = False, @@ -1129,19 +1215,21 @@ def certbot_new( server_name: Optional[str] = None, ) -> int: # Generate new certificate using certbot. - # Main function to request SSL/TLS certificates from a certificate authority - # using the ACME protocol via certbot. + # Main function to request SSL/TLS certificates from a certificate + # authority using the ACME protocol via certbot. if isinstance(credentials_path, str): credentials_path = Path(credentials_path) - ca_config = get_certificate_authority_config(ca_provider, staging) + ca_config: Dict[str, str] = get_certificate_authority_config( + ca_provider, staging + ) debug_log(LOGGER, f"Building certbot command for {domains}") debug_log(LOGGER, f"CA config: {ca_config}") debug_log(LOGGER, f"Challenge type: {challenge_type}") debug_log(LOGGER, f"Profile: {profile}") - command = [ + command: List[str] = [ CERTBOT_BIN, "certonly", "--config-dir", @@ -1190,7 +1278,8 @@ def certbot_new( # Use P-256 elliptic curve for Let's Encrypt certificates command.extend(["--elliptic-curve", "secp256r1"]) - debug_log(LOGGER, f"Using Let's Encrypt P-256 curve for {domains}") + debug_log(LOGGER, + f"Using Let's Encrypt P-256 curve for {domains}") LOGGER.info(f"Using Let's Encrypt P-256 curve for {domains}") # Handle ZeroSSL EAB credentials @@ -1199,12 +1288,16 @@ def certbot_new( LOGGER.info(f"ZeroSSL detected as CA provider for {domains}") # Check for manually provided EAB credentials first - eab_kid_env = (getenv("ACME_ZEROSSL_EAB_KID", "") or - (getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "") - if server_name else "")) - eab_hmac_env = (getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or - (getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") - if server_name else "")) + eab_kid_env: str = ( + getenv("ACME_ZEROSSL_EAB_KID", "") or + (getenv(f"{server_name}_ACME_ZEROSSL_EAB_KID", "") + if server_name else "") + ) + eab_hmac_env: str = ( + getenv("ACME_ZEROSSL_EAB_HMAC_KEY", "") or + (getenv(f"{server_name}_ACME_ZEROSSL_EAB_HMAC_KEY", "") + if server_name else "") + ) debug_log(LOGGER, "Manual EAB credentials check:") debug_log(LOGGER, f" EAB KID provided: {bool(eab_kid_env)}") @@ -1212,20 +1305,22 @@ def certbot_new( if eab_kid_env and eab_hmac_env: LOGGER.info("✓ Using manually provided ZeroSSL EAB credentials " - "from environment") + "from environment") command.extend(["--eab-kid", eab_kid_env, "--eab-hmac-key", - eab_hmac_env]) + eab_hmac_env]) LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid_env[:10]}...") elif api_key: debug_log(LOGGER, f"ZeroSSL API key provided, setting up EAB " - f"credentials") + f"credentials") LOGGER.info(f"ZeroSSL API key provided, setting up EAB " - f"credentials") + f"credentials") + eab_kid: Optional[str] + eab_hmac: Optional[str] eab_kid, eab_hmac = setup_zerossl_eab_credentials(email, api_key) if eab_kid and eab_hmac: command.extend(["--eab-kid", eab_kid, "--eab-hmac-key", - eab_hmac]) + eab_hmac]) LOGGER.info(f"✓ Using ZeroSSL EAB credentials for {domains}") LOGGER.info(f"EAB Kid: {eab_kid[:10]}...") else: @@ -1235,7 +1330,7 @@ def certbot_new( "ACME_ZEROSSL_EAB_HMAC_KEY environment variables" ) LOGGER.warning("Proceeding without EAB - this will likely " - "fail") + "fail") else: LOGGER.error("❌ No ZeroSSL API key provided!") LOGGER.error("Set ACME_ZEROSSL_API_KEY environment variable") @@ -1261,13 +1356,13 @@ def certbot_new( ) else: command.extend([f"--dns-{provider}-propagation-seconds", - propagation]) + propagation]) debug_log(LOGGER, f"Set DNS propagation time to " - f"{propagation} seconds") + f"{propagation} seconds") if provider == "route53": debug_log(LOGGER, "Route53 provider - setting environment " - "variables") + "variables") if credentials_path: with open(credentials_path, "r") as file: for line in file: @@ -1280,17 +1375,18 @@ def certbot_new( else: if credentials_path: command.extend([f"--dns-{provider}-credentials", - str(credentials_path)]) + str(credentials_path)]) - if provider in ("desec", "infomaniak", "ionos", "njalla", "scaleway"): + if provider in ("desec", "infomaniak", "ionos", "njalla", + "scaleway"): command.extend(["--authenticator", f"dns-{provider}"]) debug_log(LOGGER, f"Using explicit authenticator for {provider}") else: command.append(f"--dns-{provider}") elif challenge_type == "http": - auth_hook = JOBS_PATH.joinpath('certbot-auth.py') - cleanup_hook = JOBS_PATH.joinpath('certbot-cleanup.py') + auth_hook: Any = JOBS_PATH.joinpath('certbot-auth.py') + cleanup_hook: Any = JOBS_PATH.joinpath('certbot-cleanup.py') debug_log(LOGGER, "HTTP challenge configuration:") debug_log(LOGGER, f" Auth hook: {auth_hook}") debug_log(LOGGER, f" Cleanup hook: {cleanup_hook}") @@ -1300,9 +1396,9 @@ def certbot_new( "--manual", "--preferred-challenges=http", "--manual-auth-hook", - str(JOBS_PATH.joinpath("certbot-auth.py")), + str(auth_hook), "--manual-cleanup-hook", - str(JOBS_PATH.joinpath("certbot-cleanup.py")), + str(cleanup_hook), ] ) @@ -1310,14 +1406,16 @@ def certbot_new( command.append("--force-renewal") debug_log(LOGGER, "Force renewal enabled") - if getenv("CUSTOM_LOG_LEVEL", getenv("LOG_LEVEL", "INFO")).upper() == "DEBUG": + log_level: str = getenv("CUSTOM_LOG_LEVEL", + getenv("LOG_LEVEL", "INFO")) + if log_level.upper() == "DEBUG": command.append("-v") debug_log(LOGGER, "Verbose mode enabled for certbot") LOGGER.info(f"Executing certbot command for {domains}") # Show command but mask sensitive EAB values for security - safe_command = [] - mask_next = False + safe_command: List[str] = [] + mask_next: bool = False for item in command: if mask_next: safe_command.append("***MASKED***") @@ -1331,19 +1429,23 @@ def certbot_new( debug_log(LOGGER, f"Command: {' '.join(safe_command)}") debug_log(LOGGER, f"Environment variables: {len(working_env)} items") for key in working_env.keys(): - is_sensitive = any(sensitive in key.lower() - for sensitive in ['key', 'secret', 'token']) - value_display = '***MASKED***' if is_sensitive else 'set' + is_sensitive: bool = any( + sensitive in key.lower() + for sensitive in ['key', 'secret', 'token'] + ) + value_display: str = '***MASKED***' if is_sensitive else 'set' debug_log(LOGGER, f" {key}: {value_display}") LOGGER.info(f"Command: {' '.join(safe_command)}") - current_date = datetime.now() + current_date: datetime = datetime.now() debug_log(LOGGER, "Starting certbot process") - process = Popen(command, stdin=DEVNULL, stderr=PIPE, - universal_newlines=True, env=working_env) + process: Popen[str] = Popen( + command, stdin=DEVNULL, stderr=PIPE, + universal_newlines=True, env=working_env + ) - lines_processed = 0 + lines_processed: int = 0 while process.poll() is None: if process.stderr: rlist, _, _ = select([process.stderr], [], [], 2) @@ -1354,7 +1456,7 @@ def certbot_new( break if datetime.now() - current_date > timedelta(seconds=5): - challenge_info = ( + challenge_info: str = ( " (this may take a while depending on the provider)" if challenge_type == "dns" else "" ) @@ -1365,9 +1467,12 @@ def certbot_new( current_date = datetime.now() debug_log(LOGGER, f"Certbot still running, processed " - f"{lines_processed} output lines") + f"{lines_processed} output lines") - final_return_code = process.returncode + final_return_code: Optional[int] = process.returncode + if final_return_code is None: + final_return_code = 1 + debug_log(LOGGER, f"Certbot process completed with return code: " f"{final_return_code}") debug_log(LOGGER, f"Total output lines processed: {lines_processed}") @@ -1376,14 +1481,12 @@ def certbot_new( # Global configuration and setup -IS_MULTISITE = getenv("MULTISITE", "no") == "yes" +IS_MULTISITE: bool = getenv("MULTISITE", "no") == "yes" try: # Main execution block for certificate generation - servers = getenv("SERVER_NAME", "www.example.com").lower() or [] - - if isinstance(servers, str): - servers = servers.split(" ") + servers_env: str = getenv("SERVER_NAME", "www.example.com").lower() or "" + servers: List[str] = servers_env.split(" ") if servers_env else [] debug_log(LOGGER, "Server configuration detected:") debug_log(LOGGER, f" Multisite mode: {IS_MULTISITE}") @@ -1394,13 +1497,17 @@ def certbot_new( LOGGER.warning("There are no server names, skipping generation...") sys_exit(0) - use_letsencrypt = False - use_letsencrypt_dns = False + use_letsencrypt: bool = False + use_letsencrypt_dns: bool = False + domains_server_names: Dict[str, str] if not IS_MULTISITE: use_letsencrypt = getenv("AUTO_LETS_ENCRYPT", "no") == "yes" - use_letsencrypt_dns = getenv("LETS_ENCRYPT_CHALLENGE", "http") == "dns" - domains_server_names = {servers[0]: " ".join(servers).lower()} + use_letsencrypt_dns = ( + getenv("LETS_ENCRYPT_CHALLENGE", "http") == "dns" + ) + all_servers: str = " ".join(servers).lower() + domains_server_names = {servers[0]: all_servers} debug_log(LOGGER, "Single-site configuration:") debug_log(LOGGER, f" Let's Encrypt enabled: {use_letsencrypt}") @@ -1409,17 +1516,17 @@ def certbot_new( domains_server_names = {} for first_server in servers: - auto_le_env = f"{first_server}_AUTO_LETS_ENCRYPT" + auto_le_env: str = f"{first_server}_AUTO_LETS_ENCRYPT" if (first_server and getenv(auto_le_env, "no") == "yes"): use_letsencrypt = True - challenge_env = f"{first_server}_LETS_ENCRYPT_CHALLENGE" + challenge_env: str = f"{first_server}_LETS_ENCRYPT_CHALLENGE" if (first_server and getenv(challenge_env, "http") == "dns"): use_letsencrypt_dns = True - server_name_env = f"{first_server}_SERVER_NAME" + server_name_env: str = f"{first_server}_SERVER_NAME" domains_server_names[first_server] = getenv( server_name_env, first_server ).lower() @@ -1435,34 +1542,12 @@ def certbot_new( LOGGER.info("Let's Encrypt is not activated, skipping generation...") sys_exit(0) - provider_classes = {} + provider_classes: Dict[str, Any] = {} if use_letsencrypt_dns: debug_log(LOGGER, "DNS challenge detected - loading provider classes") - provider_classes: Dict[ - str, - Union[ - Type[CloudflareProvider], - Type[DesecProvider], - Type[DigitalOceanProvider], - Type[DnsimpleProvider], - Type[DnsMadeEasyProvider], - Type[GehirnProvider], - Type[GoogleProvider], - Type[InfomaniakProvider], - Type[IonosProvider], - Type[LinodeProvider], - Type[LuaDnsProvider], - Type[NjallaProvider], - Type[NSOneProvider], - Type[OvhProvider], - Type[Rfc2136Provider], - Type[Route53Provider], - Type[SakuraCloudProvider], - Type[ScalewayProvider], - ], - ] = { + provider_classes = { "cloudflare": CloudflareProvider, "desec": DesecProvider, "digitalocean": DigitalOceanProvider, @@ -1483,7 +1568,7 @@ def certbot_new( "scaleway": ScalewayProvider, } - JOB = Job(LOGGER, __file__) + JOB: Any = Job(LOGGER, __file__) # Restore data from db cache of certbot-renew job debug_log(LOGGER, "Restoring certificate data from database cache") @@ -1502,7 +1587,7 @@ def certbot_new( env["PYTHONPATH"] = env["PYTHONPATH"] + ( f":{DEPS_PATH}" if DEPS_PATH not in env["PYTHONPATH"] else "" ) - database_uri = getenv("DATABASE_URI") + database_uri: Optional[str] = getenv("DATABASE_URI") if database_uri: # Only assign if not None and not empty # Explicit assignment with type safety env_key: str = "DATABASE_URI" @@ -1529,13 +1614,13 @@ def certbot_new( env=env, check=False, ) - stdout = proc.stdout + stdout: str = proc.stdout or "" - WILDCARD_GENERATOR = WildcardGenerator() - credential_paths = set() - generated_domains = set() - domains_to_ask = {} - active_cert_names = set() + WILDCARD_GENERATOR: Any = WildcardGenerator() + credential_paths: Set[Any] = set() + generated_domains: Set[str] = set() + domains_to_ask: Dict[str, int] = {} + active_cert_names: Set[str] = set() if proc.returncode != 0: LOGGER.error(f"Error while checking certificates:\n{proc.stdout}") @@ -1544,51 +1629,57 @@ def certbot_new( debug_log(LOGGER, "Certificate listing successful - analyzing " "existing certificates") - certificate_blocks = stdout.split("Certificate Name: ")[1:] + certificate_blocks: List[str] = stdout.split("Certificate Name: ")[1:] debug_log(LOGGER, f"Found {len(certificate_blocks)} existing " f"certificates") for first_server, domains in domains_server_names.items(): - auto_le_check = (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") - if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) + auto_le_check: str = ( + getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") + if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no") + ) if auto_le_check != "yes": continue - challenge_check = ( + challenge_check: str = ( getenv(f"{first_server}_LETS_ENCRYPT_CHALLENGE", "http") if IS_MULTISITE else getenv("LETS_ENCRYPT_CHALLENGE", "http") ) - original_first_server = deepcopy(first_server) + original_first_server: str = deepcopy(first_server) debug_log(LOGGER, f"Processing server: {first_server}") debug_log(LOGGER, f" Challenge: {challenge_check}") debug_log(LOGGER, f" Domains: {domains}") - wildcard_check = ( + wildcard_check: str = ( getenv(f"{original_first_server}_USE_LETS_ENCRYPT_WILDCARD", - "no") + "no") if IS_MULTISITE else getenv("USE_LETS_ENCRYPT_WILDCARD", "no") ) + + domains_set: Set[str] if (challenge_check == "dns" and wildcard_check == "yes"): debug_log(LOGGER, f"Using wildcard mode for {first_server}") - wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( - (first_server,) + wildcard_domains: List[str] = ( + WILDCARD_GENERATOR.extract_wildcards_from_domains( + (first_server,) + ) ) - first_server = wildcards[0].lstrip("*.") - domains = set(wildcards) + first_server = wildcard_domains[0].lstrip("*.") + domains_set = set(wildcard_domains) else: - domains = set(str(domains).split(" ")) + domains_set = set(str(domains).split(" ")) # Add the certificate name to our active set regardless # if we're generating it or not active_cert_names.add(first_server) - certificate_block = None + certificate_block: Optional[str] = None for block in certificate_blocks: if block.startswith(f"{first_server}\n"): certificate_block = block @@ -1604,7 +1695,7 @@ def certbot_new( # Validating the credentials try: - cert_domains = search( + cert_domains_match = search( r"Domains: (?P.*)\n\s*Expiry Date: " r"(?P.*)\n", certificate_block, @@ -1618,29 +1709,37 @@ def certbot_new( ) continue - if not cert_domains: + if not cert_domains_match: LOGGER.error( f"[{original_first_server}] Failed to parse domains " "and expiry date from certificate block." ) continue - cert_domains_list = cert_domains.group("domains").strip().split() - cert_domains_set = set(cert_domains_list) - desired_domains_set = ( - set(domains) if isinstance(domains, (list, set)) - else set(str(domains).split()) + cert_domains_list: List[str] = ( + cert_domains_match.group("domains").strip().split() + ) + cert_domains_set: Set[str] = set(cert_domains_list) + desired_domains_set: Set[str] = ( + set(domains_set) if isinstance(domains_set, (list, set)) + else set(str(domains_set).split()) ) debug_log(LOGGER, f"Certificate domain comparison for " - f"{first_server}:") - debug_log(LOGGER, f" Existing: {sorted(str(d) for d in cert_domains_set)}") - debug_log(LOGGER, f" Desired: {sorted(str(d) for d in desired_domains_set)}") + f"{first_server}:") + debug_log(LOGGER, + f" Existing: {sorted(str(d) for d in cert_domains_set)}") + debug_log(LOGGER, + f" Desired: {sorted(str(d) for d in desired_domains_set)}") if cert_domains_set != desired_domains_set: domains_to_ask[first_server] = 2 - existing_sorted = sorted(str(d) for d in cert_domains_set) - desired_sorted = sorted(str(d) for d in desired_domains_set) + existing_sorted: List[str] = sorted( + str(d) for d in cert_domains_set + ) + desired_sorted: List[str] = sorted( + str(d) for d in desired_domains_set + ) LOGGER.warning( f"[{original_first_server}] Domains for {first_server} " f"differ from desired set (existing: {existing_sorted}, " @@ -1649,30 +1748,33 @@ def certbot_new( continue # Check if CA provider has changed - ca_provider_env = ( + ca_provider_env: str = ( f"{original_first_server}_ACME_SSL_CA_PROVIDER" if IS_MULTISITE else "ACME_SSL_CA_PROVIDER" ) - ca_provider = getenv(ca_provider_env, "letsencrypt") + ca_provider: str = getenv(ca_provider_env, "letsencrypt") - renewal_file = DATA_PATH.joinpath("renewal", f"{first_server}.conf") + renewal_file: Any = DATA_PATH.joinpath("renewal", + f"{first_server}.conf") if renewal_file.is_file(): - current_server = None + current_server: Optional[str] = None with renewal_file.open("r") as file: for line in file: if line.startswith("server"): current_server = line.strip().split("=", 1)[1].strip() break - staging_env = ( + staging_env: str = ( f"{original_first_server}_USE_LETS_ENCRYPT_STAGING" if IS_MULTISITE else "USE_LETS_ENCRYPT_STAGING" ) - staging_mode = getenv(staging_env, "no") == "yes" - expected_config = get_certificate_authority_config( - ca_provider, staging_mode + staging_mode: bool = getenv(staging_env, "no") == "yes" + expected_config: Dict[str, str] = ( + get_certificate_authority_config( + ca_provider, staging_mode + ) ) debug_log(LOGGER, f"CA server comparison for {first_server}:") @@ -1694,15 +1796,19 @@ def certbot_new( if IS_MULTISITE else "USE_LETS_ENCRYPT_STAGING" ) - use_staging = getenv(staging_env, "no") == "yes" - is_test_cert = "TEST_CERT" in cert_domains.group("expiry_date") + use_staging: bool = getenv(staging_env, "no") == "yes" + is_test_cert: bool = ( + "TEST_CERT" in cert_domains_match.group("expiry_date") + ) debug_log(LOGGER, f"Staging environment check for {first_server}:") debug_log(LOGGER, f" Use staging: {use_staging}") debug_log(LOGGER, f" Is test cert: {is_test_cert}") - staging_mismatch = ((is_test_cert and not use_staging) or - (not is_test_cert and use_staging)) + staging_mismatch: bool = ( + (is_test_cert and not use_staging) or + (not is_test_cert and use_staging) + ) if staging_mismatch: domains_to_ask[first_server] = 2 LOGGER.warning( @@ -1712,12 +1818,12 @@ def certbot_new( ) continue - provider_env = ( + provider_env: str = ( f"{original_first_server}_LETS_ENCRYPT_DNS_PROVIDER" if IS_MULTISITE else "LETS_ENCRYPT_DNS_PROVIDER" ) - provider = getenv(provider_env, "") + provider: str = getenv(provider_env, "") if not renewal_file.is_file(): LOGGER.error( @@ -1727,7 +1833,7 @@ def certbot_new( domains_to_ask[first_server] = 1 continue - current_provider = None + current_provider: Optional[str] = None with renewal_file.open("r") as file: for line in file: if line.startswith("authenticator"): @@ -1752,14 +1858,14 @@ def certbot_new( # Check if DNS credentials have changed if provider and current_provider == provider: debug_log(LOGGER, f"Checking DNS credentials for " - f"{first_server}") + f"{first_server}") - credential_key = ( + credential_key: str = ( f"{original_first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" ) - current_credential_items = {} + current_credential_items: Dict[str, str] = {} for env_key, env_value in environ.items(): if env_value and env_key.startswith(credential_key): @@ -1767,7 +1873,7 @@ def certbot_new( current_credential_items["json_data"] = env_value continue key, value = env_value.split(" ", 1) - cleaned_value = ( + cleaned_value: str = ( value.removeprefix("= ").replace("\\n", "\n") .replace("\\t", "\t").replace("\\r", "\r") .strip() @@ -1776,17 +1882,19 @@ def certbot_new( if "json_data" in current_credential_items: value = current_credential_items.pop("json_data") - is_base64_like = (not current_credential_items and - len(value) % 4 == 0 and - match(r"^[A-Za-z0-9+/=]+$", value)) + is_base64_like: bool = ( + not current_credential_items and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value) is not None + ) if is_base64_like: with suppress(BaseException): - decoded = b64decode(value).decode("utf-8") - json_data = loads(decoded) + decoded: str = b64decode(value).decode("utf-8") + json_data: Dict[str, Any] = loads(decoded) if isinstance(json_data, dict): - new_items = {} + new_items: Dict[str, str] = {} for k, v in json_data.items(): - cleaned_v = ( + cleaned_v: str = ( str(v).removeprefix("= ") .replace("\\n", "\n") .replace("\\t", "\t") @@ -1798,41 +1906,50 @@ def certbot_new( if current_credential_items: for key, value in current_credential_items.items(): - is_base64_candidate = (provider != "rfc2136" and - len(value) % 4 == 0 and - match(r"^[A-Za-z0-9+/=]+$", value)) + is_base64_candidate: bool = ( + provider != "rfc2136" and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value) is not None + ) if is_base64_candidate: with suppress(BaseException): decoded = b64decode(value).decode("utf-8") if decoded != value: - cleaned_decoded = ( + cleaned_decoded: str = ( decoded.removeprefix("= ") .replace("\\n", "\n") .replace("\\t", "\t") .replace("\\r", "\r") .strip() ) - current_credential_items[key] = cleaned_decoded + current_credential_items[key] = ( + cleaned_decoded + ) if provider in provider_classes: with suppress(ValidationError, KeyError): - provider_instance = provider_classes[provider]( + provider_instance: Any = provider_classes[provider]( **current_credential_items ) - current_credentials_content = ( + current_credentials_content: bytes = ( provider_instance.get_formatted_credentials() ) - file_type = provider_instance.get_file_type() - stored_credentials_path = CACHE_PATH.joinpath( - first_server, f"credentials.{file_type}" + file_type: str = ( + provider_instance.get_file_type() + ) + stored_credentials_path: Any = ( + CACHE_PATH.joinpath( + first_server, + f"credentials.{file_type}" + ) ) if stored_credentials_path.is_file(): - stored_credentials_content = ( + stored_credentials_content: bytes = ( stored_credentials_path.read_bytes() ) - content_differs = ( + content_differs: bool = ( stored_credentials_content != current_credentials_content ) @@ -1857,21 +1974,23 @@ def certbot_new( domains_to_ask[first_server] = 0 LOGGER.info( f"[{original_first_server}] Certificates already exist for " - f"domain(s) {domains}, expiry date: " - f"{cert_domains.group('expiry_date')}" + f"domain(s) {domains_set}, expiry date: " + f"{cert_domains_match.group('expiry_date')}" ) - psl_lines = None - psl_rules = None + psl_lines: Optional[List[str]] = None + psl_rules: Optional[Dict[str, Set[str]]] = None - certificates_generated = 0 - certificates_failed = 0 + certificates_generated: int = 0 + certificates_failed: int = 0 # Process each server configuration for first_server, domains in domains_server_names.items(): - auto_le_check = (getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") - if IS_MULTISITE - else getenv("AUTO_LETS_ENCRYPT", "no")) + auto_le_check = ( + getenv(f"{first_server}_AUTO_LETS_ENCRYPT", "no") + if IS_MULTISITE + else getenv("AUTO_LETS_ENCRYPT", "no") + ) if auto_le_check != "yes": LOGGER.info( f"SSL certificate generation is not activated for " @@ -1880,39 +1999,61 @@ def certbot_new( continue # Getting all the necessary data - email_env = (f"{first_server}_EMAIL_LETS_ENCRYPT" if IS_MULTISITE - else "EMAIL_LETS_ENCRYPT") - challenge_env = (f"{first_server}_LETS_ENCRYPT_CHALLENGE" - if IS_MULTISITE - else "LETS_ENCRYPT_CHALLENGE") - staging_env = (f"{first_server}_USE_LETS_ENCRYPT_STAGING" - if IS_MULTISITE - else "USE_LETS_ENCRYPT_STAGING") - wildcard_env = (f"{first_server}_USE_LETS_ENCRYPT_WILDCARD" - if IS_MULTISITE - else "USE_LETS_ENCRYPT_WILDCARD") - provider_env = (f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER" - if IS_MULTISITE - else "LETS_ENCRYPT_DNS_PROVIDER") - propagation_env = (f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION" - if IS_MULTISITE - else "LETS_ENCRYPT_DNS_PROPAGATION") - profile_env = (f"{first_server}_LETS_ENCRYPT_PROFILE" - if IS_MULTISITE - else "LETS_ENCRYPT_PROFILE") - psl_env = (f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES" - if IS_MULTISITE - else "LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES") - retries_env = (f"{first_server}_LETS_ENCRYPT_MAX_RETRIES" - if IS_MULTISITE - else "LETS_ENCRYPT_MAX_RETRIES") - ca_env = (f"{first_server}_ACME_SSL_CA_PROVIDER" if IS_MULTISITE - else "ACME_SSL_CA_PROVIDER") - api_key_env = (f"{first_server}_ACME_ZEROSSL_API_KEY" - if IS_MULTISITE - else "ACME_ZEROSSL_API_KEY") - - data = { + email_env: str = ( + f"{first_server}_EMAIL_LETS_ENCRYPT" if IS_MULTISITE + else "EMAIL_LETS_ENCRYPT" + ) + challenge_env = ( + f"{first_server}_LETS_ENCRYPT_CHALLENGE" + if IS_MULTISITE + else "LETS_ENCRYPT_CHALLENGE" + ) + staging_env = ( + f"{first_server}_USE_LETS_ENCRYPT_STAGING" + if IS_MULTISITE + else "USE_LETS_ENCRYPT_STAGING" + ) + wildcard_env = ( + f"{first_server}_USE_LETS_ENCRYPT_WILDCARD" + if IS_MULTISITE + else "USE_LETS_ENCRYPT_WILDCARD" + ) + provider_env = ( + f"{first_server}_LETS_ENCRYPT_DNS_PROVIDER" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_PROVIDER" + ) + propagation_env = ( + f"{first_server}_LETS_ENCRYPT_DNS_PROPAGATION" + if IS_MULTISITE + else "LETS_ENCRYPT_DNS_PROPAGATION" + ) + profile_env = ( + f"{first_server}_LETS_ENCRYPT_PROFILE" + if IS_MULTISITE + else "LETS_ENCRYPT_PROFILE" + ) + psl_env = ( + f"{first_server}_LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES" + if IS_MULTISITE + else "LETS_ENCRYPT_DISABLE_PUBLIC_SUFFIXES" + ) + retries_env = ( + f"{first_server}_LETS_ENCRYPT_MAX_RETRIES" + if IS_MULTISITE + else "LETS_ENCRYPT_MAX_RETRIES" + ) + ca_env = ( + f"{first_server}_ACME_SSL_CA_PROVIDER" if IS_MULTISITE + else "ACME_SSL_CA_PROVIDER" + ) + api_key_env = ( + f"{first_server}_ACME_ZEROSSL_API_KEY" + if IS_MULTISITE + else "ACME_ZEROSSL_API_KEY" + ) + + server_data: Dict[str, Any] = { "email": (getenv(email_env, "") or f"contact@{first_server}"), "challenge": getenv(challenge_env, "http"), "staging": getenv(staging_env, "no") == "yes", @@ -1927,36 +2068,38 @@ def certbot_new( "credential_items": {}, } - debug_log(LOGGER, f"Service {first_server} configuration: {data}") + debug_log(LOGGER, f"Service {first_server} configuration: {server_data}") LOGGER.info(f"Service {first_server} configuration:") - LOGGER.info(f" CA Provider: {data['ca_provider']}") - api_key_status = 'Yes' if data['api_key'] else 'No' + LOGGER.info(f" CA Provider: {server_data['ca_provider']}") + api_key_status: str = 'Yes' if server_data['api_key'] else 'No' LOGGER.info(f" API Key provided: {api_key_status}") - LOGGER.info(f" Challenge type: {data['challenge']}") - LOGGER.info(f" Staging: {data['staging']}") - LOGGER.info(f" Wildcard: {data['use_wildcard']}") + LOGGER.info(f" Challenge type: {server_data['challenge']}") + LOGGER.info(f" Staging: {server_data['staging']}") + LOGGER.info(f" Wildcard: {server_data['use_wildcard']}") # Override profile if custom profile is set - custom_profile_env = (f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE" - if IS_MULTISITE - else "LETS_ENCRYPT_CUSTOM_PROFILE") - custom_profile = getenv(custom_profile_env, "").strip() - if custom_profile: - data["profile"] = custom_profile - debug_log(LOGGER, f"Using custom profile: {custom_profile}") - - if data["challenge"] == "http" and data["use_wildcard"]: + custom_profile_env: str = ( + f"{first_server}_LETS_ENCRYPT_CUSTOM_PROFILE" + if IS_MULTISITE + else "LETS_ENCRYPT_CUSTOM_PROFILE" + ) + custom_profile_str: str = getenv(custom_profile_env, "").strip() + if custom_profile_str: + server_data["profile"] = custom_profile_str + debug_log(LOGGER, f"Using custom profile: {custom_profile_str}") + + if server_data["challenge"] == "http" and server_data["use_wildcard"]: LOGGER.warning( f"Wildcard is not supported with HTTP challenge, " f"disabling wildcard for service {first_server}..." ) - data["use_wildcard"] = False + server_data["use_wildcard"] = False - should_skip_cert_check = ( - (not data["use_wildcard"] and + should_skip_cert_check: bool = ( + (not server_data["use_wildcard"] and not domains_to_ask.get(first_server)) or - (data["use_wildcard"] and not domains_to_ask.get( + (server_data["use_wildcard"] and not domains_to_ask.get( WILDCARD_GENERATOR.extract_wildcards_from_domains( (first_server,) )[0].lstrip("*.") @@ -1966,25 +2109,26 @@ def certbot_new( debug_log(LOGGER, f"No certificate needed for {first_server}") continue - if not data["max_retries"].isdigit(): + if not server_data["max_retries"].isdigit(): LOGGER.warning( f"Invalid max retries value for service {first_server}: " - f"{data['max_retries']}, using default value of 0..." + f"{server_data['max_retries']}, using default value of 0..." ) - data["max_retries"] = 0 + server_data["max_retries"] = 0 else: - data["max_retries"] = int(data["max_retries"]) + server_data["max_retries"] = int(server_data["max_retries"]) # Getting the DNS provider data if necessary - if data["challenge"] == "dns": - debug_log(LOGGER, f"Processing DNS credentials for {first_server}") + if server_data["challenge"] == "dns": + debug_log(LOGGER, + f"Processing DNS credentials for {first_server}") credential_key = ( f"{first_server}_LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" if IS_MULTISITE else "LETS_ENCRYPT_DNS_CREDENTIAL_ITEM" ) - credential_items = {} + credential_items: Dict[str, str] = {} # Collect all credential items for env_key, env_value in environ.items(): @@ -2005,9 +2149,11 @@ def certbot_new( value = credential_items.pop("json_data") # Handle the case of a single credential that might be # base64-encoded JSON - is_potential_json = (not credential_items and - len(value) % 4 == 0 and - match(r"^[A-Za-z0-9+/=]+$", value)) + is_potential_json: bool = ( + not credential_items and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value) is not None + ) if is_potential_json: try: decoded = b64decode(value).decode("utf-8") @@ -2022,7 +2168,7 @@ def certbot_new( .replace("\\r", "\r").strip() ) new_items[k.lower()] = cleaned_v - data["credential_items"] = new_items + server_data["credential_items"] = new_items except BaseException as e: LOGGER.debug(format_exc()) LOGGER.error( @@ -2030,14 +2176,16 @@ def certbot_new( f"{first_server}: {value} : \n{e}" ) - if not data["credential_items"]: + if not server_data["credential_items"]: # Process regular credentials - data["credential_items"] = {} + server_data["credential_items"] = {} for key, value in credential_items.items(): # Check for base64 encoding - is_base64_candidate = (data["provider"] != "rfc2136" and - len(value) % 4 == 0 and - match(r"^[A-Za-z0-9+/=]+$", value)) + is_base64_candidate = ( + server_data["provider"] != "rfc2136" and + len(value) % 4 == 0 and + match(r"^[A-Za-z0-9+/=]+$", value) is not None + ) if is_base64_candidate: try: decoded = b64decode(value).decode("utf-8") @@ -2054,55 +2202,61 @@ def certbot_new( f"Error while decoding credential item {key} " f"for service {first_server}: {value} : \n{e}" ) - data["credential_items"][key] = value + server_data["credential_items"][key] = value - safe_data = data.copy() - masked_items = {k: "***MASKED***" - for k in data["credential_items"].keys()} + safe_data: Dict[str, Any] = server_data.copy() + masked_items: Dict[str, str] = { + k: "***MASKED***" + for k in server_data["credential_items"].keys() + } safe_data["credential_items"] = masked_items - if data["api_key"]: + if server_data["api_key"]: safe_data["api_key"] = "***MASKED***" debug_log(LOGGER, f"Safe data for service {first_server}: " f"{dumps(safe_data)}") # Validate CA provider and API key requirements - api_key_status = 'Yes' if data['api_key'] else 'No' + api_key_status = 'Yes' if server_data['api_key'] else 'No' LOGGER.info( - f"Service {first_server} - CA Provider: {data['ca_provider']}, " + f"Service {first_server} - CA Provider: {server_data['ca_provider']}, " f"API Key provided: {api_key_status}" ) - if data["ca_provider"].lower() == "zerossl": - if not data["api_key"]: + if server_data["ca_provider"].lower() == "zerossl": + if not server_data["api_key"]: LOGGER.warning( f"ZeroSSL API key not provided for service " f"{first_server}, falling back to Let's Encrypt..." ) - data["ca_provider"] = "letsencrypt" + server_data["ca_provider"] = "letsencrypt" else: LOGGER.info(f"✓ ZeroSSL configuration valid for service " - f"{first_server}") + f"{first_server}") # Checking if the DNS data is valid - if data["challenge"] == "dns": - if not data["provider"]: - available_providers = ', '.join(str(p) for p in provider_classes.keys()) + if server_data["challenge"] == "dns": + if not server_data["provider"]: + available_providers: str = ', '.join( + str(p) for p in provider_classes.keys() + ) LOGGER.warning( f"No provider found for service {first_server} " f"(available providers: {available_providers}), " "skipping certificate(s) generation..." ) continue - elif data["provider"] not in provider_classes: - available_providers = ', '.join(str(p) for p in provider_classes.keys()) + elif server_data["provider"] not in provider_classes: + available_providers = ', '.join( + str(p) for p in provider_classes.keys() + ) LOGGER.warning( - f"Provider {data['provider']} not found for service " + f"Provider {server_data['provider']} not found for service " f"{first_server} (available providers: " f"{available_providers}), skipping certificate(s) " f"generation..." ) continue - elif not data["credential_items"]: + elif not server_data["credential_items"]: LOGGER.warning( f"No valid credentials items found for service " f"{first_server} (you should have at least one), " @@ -2111,8 +2265,8 @@ def certbot_new( continue try: - provider = provider_classes[data["provider"]]( - **data["credential_items"] + dns_provider_instance: Any = provider_classes[server_data["provider"]]( + **server_data["credential_items"] ) except ValidationError as ve: LOGGER.debug(format_exc()) @@ -2122,32 +2276,36 @@ def certbot_new( ) continue - content = provider.get_formatted_credentials() + content: bytes = dns_provider_instance.get_formatted_credentials() else: content = b"http_challenge" - is_blacklisted = False + is_blacklisted: bool = False # Adding the domains to Wildcard Generator if necessary - file_type = (provider.get_file_type() if data["challenge"] == "dns" - else "txt") - file_path = (first_server, f"credentials.{file_type}") + file_type_str: str = ( + dns_provider_instance.get_file_type() if server_data["challenge"] == "dns" + else "txt" + ) + file_path: Tuple[str, ...] = (first_server, + f"credentials.{file_type_str}") - if data["use_wildcard"]: + if server_data["use_wildcard"]: # Use the improved method for generating consistent group names - group = WILDCARD_GENERATOR.create_group_name( + hash_value: Any = bytes_hash(content, algorithm="sha1") + group: str = WILDCARD_GENERATOR.create_group_name( domain=first_server, - provider=(data["provider"] if data["challenge"] == "dns" - else "http"), - challenge_type=data["challenge"], - staging=data["staging"], - content_hash=bytes_hash(content, algorithm="sha1"), - profile=data["profile"], + provider=(server_data["provider"] if server_data["challenge"] == "dns" + else "http"), + challenge_type=server_data["challenge"], + staging=server_data["staging"], + content_hash=hash_value, + profile=server_data["profile"], ) - wildcard_info = ( + wildcard_info: str = ( "the propagation time will be the provider's default and " - if data["challenge"] == "dns" else "" + if server_data["challenge"] == "dns" else "" ) LOGGER.info( f"Service {first_server} is using wildcard, " @@ -2155,23 +2313,28 @@ def certbot_new( f"domain that created the group..." ) - if data["check_psl"]: + if server_data["check_psl"]: if psl_lines is None: debug_log(LOGGER, "Loading PSL for wildcard domain " - "validation") + "validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: debug_log(LOGGER, "Parsing PSL rules") psl_rules = parse_psl(psl_lines) - wildcards = WILDCARD_GENERATOR.extract_wildcards_from_domains( - str(domains).split(" ") + wildcards_list: List[str] = ( + WILDCARD_GENERATOR.extract_wildcards_from_domains( + str(domains).split(" ") + ) ) + wildcard_str: str = ', '.join( + str(w) for w in wildcards_list + ) LOGGER.info(f"Wildcard domains for {first_server}: " - f"{wildcards}") + f"{wildcard_str}") - for d in wildcards: + for d in wildcards_list: if is_domain_blacklisted(d, psl_rules): LOGGER.error( f"Wildcard domain {d} is blacklisted by Public " @@ -2183,14 +2346,15 @@ def certbot_new( if not is_blacklisted: WILDCARD_GENERATOR.extend( - group, str(domains).split(" "), data["email"], - data["staging"] + group, str(domains).split(" "), server_data["email"], + server_data["staging"] ) - file_path = (f"{group}.{file_type}",) + file_path = (f"{group}.{file_type_str}",) LOGGER.info(f"[{first_server}] Wildcard group {group}") - elif data["check_psl"]: + elif server_data["check_psl"]: if psl_lines is None: - debug_log(LOGGER, "Loading PSL for regular domain validation") + debug_log(LOGGER, + "Loading PSL for regular domain validation") psl_lines = load_public_suffix_list(JOB) if psl_rules is None: debug_log(LOGGER, "Parsing PSL rules") @@ -2210,15 +2374,18 @@ def certbot_new( continue # Generating the credentials file - credentials_path = CACHE_PATH.joinpath(*file_path) + credentials_path: Any = CACHE_PATH.joinpath(*file_path) - if data["challenge"] == "dns": - debug_log(LOGGER, f"Managing credentials file for {first_server}: " - f"{credentials_path}") + if server_data["challenge"] == "dns": + debug_log(LOGGER, + f"Managing credentials file for {first_server}: " + f"{credentials_path}") if not credentials_path.is_file(): - service_id = (first_server if not data["use_wildcard"] - else "") + service_id: str = (first_server if not server_data["use_wildcard"] + else "") + cached: Any + err: Any cached, err = JOB.cache_file( credentials_path.name, content, job_name="certbot-renew", service_id=service_id @@ -2233,27 +2400,29 @@ def certbot_new( f"Successfully saved service {first_server}'s " "credentials file in cache" ) - elif data["use_wildcard"]: + elif server_data["use_wildcard"]: LOGGER.info( f"Service {first_server}'s wildcard credentials file " "has already been generated" ) else: - old_content = credentials_path.read_bytes() + old_content: bytes = credentials_path.read_bytes() if old_content != content: LOGGER.warning( f"Service {first_server}'s credentials file is " "outdated, updating it..." ) - cached, err = JOB.cache_file( + cached_updated: Any + err_updated: Any + cached_updated, err_updated = JOB.cache_file( credentials_path.name, content, job_name="certbot-renew", service_id=first_server ) - if not cached: + if not cached_updated: LOGGER.error( f"Error while updating service {first_server}'s " - f"credentials file in cache: {err}" + f"credentials file in cache: {err_updated}" ) continue LOGGER.info( @@ -2271,37 +2440,39 @@ def certbot_new( # warnings from certbot) credentials_path.chmod(0o600) - if data["use_wildcard"]: + if server_data["use_wildcard"]: debug_log(LOGGER, f"Wildcard processing complete for " - f"{first_server}") + f"{first_server}") continue - domains = str(domains).replace(" ", ",") - ca_name = get_certificate_authority_config(data["ca_provider"])["name"] - staging_info = ' using staging' if data['staging'] else '' + domains_str: str = str(domains).replace(" ", ",") + ca_name: str = get_certificate_authority_config( + server_data["ca_provider"] + )["name"] + staging_info: str = ' using staging' if server_data['staging'] else '' LOGGER.info( - f"Asking {ca_name} certificates for domain(s): {domains} " - f"(email = {data['email']}){staging_info} " - f" with {data['challenge']} challenge, using " - f"{data['profile']!r} profile..." + f"Asking {ca_name} certificates for domain(s): {domains_str} " + f"(email = {server_data['email']}){staging_info} " + f" with {server_data['challenge']} challenge, using " + f"{server_data['profile']!r} profile..." ) - debug_log(LOGGER, f"Requesting certificate for {domains}") + debug_log(LOGGER, f"Requesting certificate for {domains_str}") - cert_result = certbot_new_with_retry( - data["challenge"], - domains, - data["email"], - data["provider"], + cert_result: int = certbot_new_with_retry( + cast(Literal["dns", "http"], server_data["challenge"]), + domains_str, + server_data["email"], + server_data["provider"], credentials_path, - data["propagation"], - data["profile"], - data["staging"], + server_data["propagation"], + server_data["profile"], + server_data["staging"], domains_to_ask[first_server] == 2, cmd_env=env, - max_retries=data["max_retries"], - ca_provider=data["ca_provider"], - api_key=data["api_key"], + max_retries=server_data["max_retries"], + ca_provider=server_data["ca_provider"], + api_key=server_data["api_key"], server_name=first_server, ) @@ -2309,69 +2480,81 @@ def certbot_new( status = 2 certificates_failed += 1 LOGGER.error(f"Certificate generation failed for domain(s) " - f"{domains}...") + f"{domains_str}...") else: status = 1 if status == 0 else status certificates_generated += 1 LOGGER.info(f"Certificate generation succeeded for domain(s): " - f"{domains}") + f"{domains_str}") - generated_domains.update(domains.split(",")) + generated_domains.update(domains_str.split(",")) # Generating the wildcards if necessary - wildcards = WILDCARD_GENERATOR.get_wildcards() - if wildcards: - debug_log(LOGGER, f"Processing {len(wildcards)} wildcard groups") + wildcard_groups: Dict[str, Any] = WILDCARD_GENERATOR.get_wildcards() + if wildcard_groups: + debug_log(LOGGER, f"Processing {len(wildcard_groups)} wildcard groups") - for group, data in wildcards.items(): - if not data: + for group, group_data in wildcard_groups.items(): + if not group_data: continue # Generating the certificate from the generated credentials - group_parts = group.split("_") - provider = group_parts[0] - profile = group_parts[2] - base_domain = group_parts[3] + group_parts: List[str] = group.split("_") + provider_name: str = group_parts[0] + profile: str = group_parts[2] + base_domain: str = group_parts[3] debug_log(LOGGER, f"Processing wildcard group: {group}") - debug_log(LOGGER, f" Provider: {provider}") + debug_log(LOGGER, f" Provider: {provider_name}") debug_log(LOGGER, f" Profile: {profile}") debug_log(LOGGER, f" Base domain: {base_domain}") - email = data.pop("email") - file_type = (provider_classes[provider].get_file_type() - if provider in provider_classes else 'txt') - credentials_file = CACHE_PATH.joinpath(f"{group}.{file_type}") + email: str = group_data.pop("email") + wildcard_file_type: str = ( + str(provider_classes[provider_name].get_file_type()) + if provider_name in provider_classes else 'txt' + ) + credentials_file: Any = CACHE_PATH.joinpath( + f"{group}.{wildcard_file_type}" + ) # Get CA provider for this group - original_server = None + original_server: Optional[str] = None for server in domains_server_names.keys(): if base_domain in server or server in base_domain: original_server = server break ca_provider = "letsencrypt" # default - api_key = None + api_key: Optional[str] = None if original_server: - ca_env = (f"{original_server}_ACME_SSL_CA_PROVIDER" - if IS_MULTISITE - else "ACME_SSL_CA_PROVIDER") + ca_env = ( + f"{original_server}_ACME_SSL_CA_PROVIDER" + if IS_MULTISITE + else "ACME_SSL_CA_PROVIDER" + ) ca_provider = getenv(ca_env, "letsencrypt") - api_key_env = (f"{original_server}_ACME_ZEROSSL_API_KEY" - if IS_MULTISITE - else "ACME_ZEROSSL_API_KEY") - api_key = getenv(api_key_env, "") + api_key_env = ( + f"{original_server}_ACME_ZEROSSL_API_KEY" + if IS_MULTISITE + else "ACME_ZEROSSL_API_KEY" + ) + api_key = getenv(api_key_env, "") or None # Process different environment types (staging/prod) - for key, domains in data.items(): + for key, domains in group_data.items(): if not domains: continue - staging = key == "staging" - ca_name = get_certificate_authority_config(ca_provider)["name"] + staging: bool = key == "staging" + ca_name = get_certificate_authority_config( + ca_provider + )["name"] staging_info = ' using staging ' if staging else '' - challenge_type = 'dns' if provider in provider_classes else 'http' + challenge_type: str = ( + 'dns' if provider_name in provider_classes else 'http' + ) LOGGER.info( f"Asking {ca_name} wildcard certificates for domain(s): " f"{domains} (email = {email}){staging_info} " @@ -2379,7 +2562,7 @@ def certbot_new( f"using {profile!r} profile..." ) - domains_split = domains.split(",") + domains_split: List[str] = domains.split(",") # Add wildcard certificate names to active set for domain in domains_split: @@ -2388,13 +2571,13 @@ def certbot_new( active_cert_names.add(base_domain) debug_log(LOGGER, f"Requesting wildcard certificate for " - f"{domains}") + f"{domains}") - wildcard_result = certbot_new_with_retry( + wildcard_result: int = certbot_new_with_retry( "dns", domains, email, - provider, + provider_name, credentials_file, "default", profile, @@ -2410,12 +2593,12 @@ def certbot_new( status = 2 certificates_failed += 1 LOGGER.error(f"Certificate generation failed for " - f"domain(s) {domains}...") + f"domain(s) {domains}...") else: status = 1 if status == 0 else status certificates_generated += 1 LOGGER.info(f"Certificate generation succeeded for " - f"domain(s): {domains}") + f"domain(s): {domains}") generated_domains.update(domains_split) else: @@ -2433,23 +2616,27 @@ def certbot_new( # Clearing all missing credentials files debug_log(LOGGER, "Cleaning up old credentials files") - cleaned_files = 0 - for ext in ("*.ini", "*.env", "*.json"): + cleaned_files: int = 0 + ext_patterns: Tuple[str, ...] = ("*.ini", "*.env", "*.json") + for ext in ext_patterns: for file in list(CACHE_PATH.rglob(ext)): if "etc" in file.parts or not file.is_file(): continue # If the file is not in the wildcard groups, remove it if file not in credential_paths: LOGGER.info(f"Removing old credentials file {file}") - service_id = (file.parent.name - if file.parent.name != "letsencrypt" else "") + service_id = ( + file.parent.name + if file.parent.name != "letsencrypt" else "" + ) JOB.del_cache( file.name, job_name="certbot-renew", service_id=service_id ) cleaned_files += 1 - debug_log(LOGGER, f"Cleaned up {cleaned_files} old credentials files") + debug_log(LOGGER, + f"Cleaned up {cleaned_files} old credentials files") # Clearing all no longer needed certificates if getenv("LETS_ENCRYPT_CLEAR_OLD_CERTS", "no") == "yes": @@ -2482,15 +2669,16 @@ def certbot_new( if proc.returncode == 0: certificate_blocks = proc.stdout.split("Certificate Name: ")[1:] - certificates_removed = 0 + certificates_removed: int = 0 - debug_log(LOGGER, f"Found {len(certificate_blocks)} certificates " - f"to evaluate") + debug_log(LOGGER, + f"Found {len(certificate_blocks)} certificates " + f"to evaluate") debug_log(LOGGER, f"Active certificates: " - f"{sorted(str(name) for name in active_cert_names)}") + f"{sorted(str(name) for name in active_cert_names)}") for block in certificate_blocks: - cert_name = block.split("\n", 1)[0].strip() + cert_name: str = block.split("\n", 1)[0].strip() # Skip certificates that are in our active list if cert_name in active_cert_names: @@ -2527,12 +2715,14 @@ def certbot_new( if delete_proc.returncode == 0: LOGGER.info(f"Successfully deleted certificate " - f"{cert_name}") + f"{cert_name}") certificates_removed += 1 - cert_dir = DATA_PATH.joinpath("live", cert_name) - archive_dir = DATA_PATH.joinpath("archive", cert_name) - renewal_file = DATA_PATH.joinpath("renewal", - f"{cert_name}.conf") + cert_dir: Any = DATA_PATH.joinpath("live", cert_name) + archive_dir: Any = DATA_PATH.joinpath("archive", + cert_name) + cert_renewal_file: Any = DATA_PATH.joinpath("renewal", + f"{cert_name}.conf") + path: Any for path in (cert_dir, archive_dir): if path.exists(): try: @@ -2540,22 +2730,24 @@ def certbot_new( try: file.unlink() except Exception as e: - LOGGER.error(f"Failed to remove file " - f"{file}: {e}") + LOGGER.error( + f"Failed to remove file " + f"{file}: {e}" + ) path.rmdir() LOGGER.info(f"Removed directory {path}") except Exception as e: LOGGER.error(f"Failed to remove directory " - f"{path}: {e}") - if renewal_file.exists(): + f"{path}: {e}") + if cert_renewal_file.exists(): try: - renewal_file.unlink() + cert_renewal_file.unlink() LOGGER.info(f"Removed renewal file " - f"{renewal_file}") + f"{cert_renewal_file}") except Exception as e: LOGGER.error( f"Failed to remove renewal file " - f"{renewal_file}: {e}" + f"{cert_renewal_file}: {e}" ) else: LOGGER.error( @@ -2564,7 +2756,7 @@ def certbot_new( ) debug_log(LOGGER, f"Certificate cleanup completed - removed " - f"{certificates_removed} certificates") + f"{certificates_removed} certificates") else: LOGGER.error(f"Error listing certificates: {proc.stdout}") @@ -2572,9 +2764,11 @@ def certbot_new( if DATA_PATH.is_dir() and list(DATA_PATH.iterdir()): debug_log(LOGGER, "Saving certificate data to database cache") - cached, err = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") - if not cached: - LOGGER.error(f"Error while saving data to db cache: {err}") + cached_final: Any + err_final: Any + cached_final, err_final = JOB.cache_dir(DATA_PATH, job_name="certbot-renew") + if not cached_final: + LOGGER.error(f"Error while saving data to db cache: {err_final}") else: LOGGER.info("Successfully saved data to db cache") debug_log(LOGGER, "Database cache update completed") @@ -2582,8 +2776,9 @@ def certbot_new( debug_log(LOGGER, "No certificate data to cache") except SystemExit as e: - status = e.code - debug_log(LOGGER, f"Script exiting via SystemExit with code: {e.code}") + exit_code: int = cast(int, e.code) + status = exit_code + debug_log(LOGGER, f"Script exiting via SystemExit with code: {exit_code}") except BaseException as e: status = 1 LOGGER.debug(format_exc())