Replace urlretrieve with streaming download using urlopen (#384)

Updated setup.py to replace urllib.request.urlretrieve with a
streaming download helper based on urlopen. This avoids the drawbacks
of urlretrieve, such as lack of streaming support and limited control
over requests. Using urlopen allows downloads to be written in chunks,
reduces memory usage, and makes it easier to extend the logic with
custom headers, retries, or checksum verification in the future. This
improves the reliability and security of fetching dependency tarballs
during the build process.

Author: mxamin

.github/workflows/manylinux.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Build linux_x86_64 wheel
         env:
           PYXMLSEC_STATIC_DEPS: true
-          PYXMLSEC_LIBXML2_VERSION: 2.14.6  # Lock it to libxml2 2.14.6 until the issue with 2.15.x is resolved; e.g. https://github.com/lsh123/xmlsec/issues/948
+          PYXMLSEC_LIBXML2_VERSION: 2.14.6  # Lock it to libxml2 2.14.6 to match it with lxml
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           /opt/python/${{ matrix.python-abi }}/bin/python -m build

setup.py

@@ -14,7 +14,7 @@
 from distutils.version import StrictVersion as Version
 from pathlib import Path
 from urllib.parse import urljoin
-from urllib.request import Request, urlcleanup, urlopen, urlretrieve
+from urllib.request import Request, urlcleanup, urlopen
 
 from setuptools import Extension, setup
 from setuptools.command.build_ext import build_ext as build_ext_orig
@@ -106,6 +106,16 @@ def latest_xmlsec_release():
     return tar_gz['browser_download_url']
 
 
+def download_lib(url, filename):
+    req = Request(url, headers={'User-Agent': 'python-xmlsec build'})
+    with urlopen(req) as r, open(filename, 'wb') as f:
+        while True:
+            chunk = r.read(8192)
+            if not chunk:
+                break
+            f.write(chunk)
+
+
 class CrossCompileInfo:
     def __init__(self, host, arch, compiler):
         self.host = host
@@ -234,7 +244,7 @@ def prepare_static_build_win(self):
             else:
                 self.info(f'Retrieving "{url}" to "{destfile}"')
                 urlcleanup()  # work around FTP bug 27973 in Py2.7.12+
-                urlretrieve(url, str(destfile))
+                download_lib(url, str(destfile))
 
         for p in self.libs_dir.glob('*.zip'):
             with zipfile.ZipFile(str(p)) as f:
@@ -297,7 +307,7 @@ def prepare_static_build(self, build_platform):
             else:
                 url = f'https://api.github.com/repos/openssl/openssl/tarball/openssl-{self.openssl_version}'
                 self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version))
-            urlretrieve(url, str(openssl_tar))
+            download_lib(url, str(openssl_tar))
 
         # fetch zlib
         zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None)
@@ -310,7 +320,7 @@ def prepare_static_build(self, build_platform):
             else:
                 url = f'https://zlib.net/fossils/zlib-{self.zlib_version}.tar.gz'
                 self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_ZLIB_VERSION={self.zlib_version}, downloading from {url}'))
-            urlretrieve(url, str(zlib_tar))
+            download_lib(url, str(zlib_tar))
 
         # fetch libiconv
         libiconv_tar = next(self.libs_dir.glob('libiconv*.tar.gz'), None)
@@ -319,13 +329,13 @@ def prepare_static_build(self, build_platform):
             libiconv_tar = self.libs_dir / 'libiconv.tar.gz'
             if self.libiconv_version is None:
                 url = latest_libiconv_release()
-                self.info('{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}'))
+                self.info('{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION unset, downloading latest from {url}'))
             else:
                 url = f'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{self.libiconv_version}.tar.gz'
                 self.info(
-                    '{:10}: {}'.format('zlib', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}')
+                    '{:10}: {}'.format('libiconv', f'PYXMLSEC_LIBICONV_VERSION={self.libiconv_version}, downloading from {url}')
                 )
-            urlretrieve(url, str(libiconv_tar))
+            download_lib(url, str(libiconv_tar))
 
         # fetch libxml2
         libxml2_tar = next(self.libs_dir.glob('libxml2*.tar.xz'), None)
@@ -341,7 +351,7 @@ def prepare_static_build(self, build_platform):
                     '{:10}: {}'.format('libxml2', f'PYXMLSEC_LIBXML2_VERSION={self.libxml2_version}, downloading from {url}')
                 )
             libxml2_tar = self.libs_dir / 'libxml2.tar.xz'
-            urlretrieve(url, str(libxml2_tar))
+            download_lib(url, str(libxml2_tar))
 
         # fetch libxslt
         libxslt_tar = next(self.libs_dir.glob('libxslt*.tar.gz'), None)
@@ -357,7 +367,7 @@ def prepare_static_build(self, build_platform):
                     '{:10}: {}'.format('libxslt', f'PYXMLSEC_LIBXSLT_VERSION={self.libxslt_version}, downloading from {url}')
                 )
             libxslt_tar = self.libs_dir / 'libxslt.tar.gz'
-            urlretrieve(url, str(libxslt_tar))
+            download_lib(url, str(libxslt_tar))
 
         # fetch xmlsec1
         xmlsec1_tar = next(self.libs_dir.glob('xmlsec1*.tar.gz'), None)
@@ -372,7 +382,7 @@ def prepare_static_build(self, build_platform):
                     '{:10}: {}'.format('xmlsec1', f'PYXMLSEC_XMLSEC1_VERSION={self.xmlsec1_version}, downloading from {url}')
                 )
             xmlsec1_tar = self.libs_dir / 'xmlsec1.tar.gz'
-            urlretrieve(url, str(xmlsec1_tar))
+            download_lib(url, str(xmlsec1_tar))
 
         for file in (openssl_tar, zlib_tar, libiconv_tar, libxml2_tar, libxslt_tar, xmlsec1_tar):
             self.info(f'Unpacking {file.name}')