OpenSSL 2023.09 updates. (#175)

* Updated brink.conf from server repo.

* Updated OpenSSL sources to version 1.1.1w.

* Updated OpenSSL 1.1.1 version to build.

* Patched OpenSSL 1.0.2 sources for CVE-2023-0286.

* OpenSSL version string fixes.

* Exclude safety checks for cryptography's OpenSSL and requests.

* Exclude one more safety check for certifi.

* Updated macOS label to use.

* Try specifying the full path to the choco binary.

* Try fixing the Python build on macOS 11.

* Exclude one more safety check for pywin32.

* Try reverting to brink.conf from master.

* Revert "Try reverting to brink.conf from master."

This reverts commit 12b1845.

* Try the py2-support branch of compat.

* Try fixing more issues on macOS 11.

* Patch cryptography for CVE-2023-23931 when built w/o pip.

* Try fixing `cryptography` 3.3.2 too for CVE-2023-23931.

* Try updating psutil to 5.9.5 on all platforms.

* Updated SQLite sources to version 3.43.1.

* Updated SQLite Windows DLLs to version 3.43.1.

* Updated SQLite version to build to latest: 3.43.1.

* Updated zlib sources to version 1.3.

* Updated SQLite version to build to latest: 1.3.

* Updated external deps sheets.

* Updated external deps sheets (bis).

* Changes after own review.

* Updated external deps sheets after own review.

* Try building the ARM64 package on Amazon 2 running on Laja.

* Try fixing the ARM64 build.

* Try fixing the ARM64 build, take two.

* Try fixing the ARM64 build, take three.

* Try fixing CVE-2021-3177 for Python.

* Try fixing CVE-2023-24329 for Python.

* CVE-2021-3177 is not fixable on Windows.

* Fix the patch for CVE-2023-24329.

* Updated external deps stuff.

* Updated comments for safety's ignored opts.

* Try sleeping 10s before hacking GHA's macOS Homebrew setup.

* Changes after own review.

.github/workflows/bare.yaml

@@ -29,8 +29,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # ARM64 is currently our virtualized Ubuntu 16.04 image.
-        runs-on: [ ubuntu-20.04, ubuntu-18.04, ARM64 ]
+        # The ARM64 build actually runs on an Amazon Docker container on Laja.
+        runs-on: [ ubuntu-20.04, ubuntu-18.04, amzn-2-arm64 ]
     timeout-minutes: 120
     steps:
     - name: Prepare OS
@@ -96,13 +96,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runs-on: [ macos-10.15 ]
+        runs-on: [ macos-11 ]
     timeout-minutes: 60
     steps:
     # Avoid linking to Homebrew's libintl during build.
     # Needed tools are to be used from /usr/bin.
     - name: Hack Homebrew
       run: |
+        sleep 10
         sudo find /usr/local -name 'libffi*' -exec chmod a-r {} +
         sudo find /usr/local -name 'libintl*' -exec chmod a-r {} +
         sudo rm -f /usr/local/bin/{wget,curl,git}
@@ -173,7 +174,7 @@ jobs:
     - name: Prepare OS
       shell: powershell
       run: |
-        chocolatey install --yes --no-progress make nasm 7zip curl
+        choco install --yes --no-progress make nasm 7zip curl
         # There's no vcpython27 choco pkg since Microsoft removed the installer.
         Start-BitsTransfer https://bin.chevah.com:20443/third-party-stuff/VCForPython27.msi
         msiexec /quiet /i VCForPython27.msi

brink.conf

@@ -1,5 +1,5 @@
-BASE_REQUIREMENTS='pip==20.3.4chevah chevah-brink==0.79.0 paver==1.2.4'
-PYTHON_CONFIGURATION='[email protected].90dc4a6'
+BASE_REQUIREMENTS='pip==20.3.4chevah1 chevah-brink==0.79.0 paver==1.2.4'
+PYTHON_CONFIGURATION='[email protected].52fb2f5'
 # For production packages there are 2 options:
 BINARY_DIST_URI='https://github.com/chevah/python-package/releases/download'
 #BINARY_DIST_URI='https://bin.chevah.com:20443/production'

brink.sh

@@ -640,14 +640,14 @@ check_glibc_version(){
 
     # Supported minimum minor glibc 2.X versions for various arches.
     # For x64, we build on CentOS 5.11 (Final) with glibc 2.5.
-    # For arm64, we build on Ubuntu 16.04 with glibc 2.23.
+    # For arm64, we build on Amazon Linux 2 with glibc 2.26.
     # Beware we haven't normalized arch names yet.
     case "$ARCH" in
         "amd64"|"x86_64"|"x64")
             supported_glibc2_version=5
             ;;
         "aarch64"|"arm64")
-            supported_glibc2_version=23
+            supported_glibc2_version=26
             ;;
         *)
             (>&2 echo "$ARCH is an unsupported arch for generic Linux!")
@@ -658,7 +658,7 @@ check_glibc_version(){
     echo "No specific runtime for the current distribution / version / arch."
     echo "Minimum glibc version for this arch: 2.${supported_glibc2_version}."
 
-    # Tested with glibc 2.5/2.11.3/2.12/2.23/2.28-31 and eglibc 2.13/2.19.
+    # Tested with glibc 2.5/2.11.3/2.12/2.23/2.26/2.28-31 and eglibc 2.13/2.19.
     glibc_version=$(head -n 1 $ldd_output_file | rev | cut -d\  -f1 | rev)
     rm $ldd_output_file
 

chevah_build

@@ -14,19 +14,19 @@ set -o pipefail   # don't ignore exit codes when piping output
 
 PYTHON_BUILD_VERSION="2.7.18"
 LIBFFI_VERSION="3.4.4"
-ZLIB_VERSION="1.2.13"
+ZLIB_VERSION="1.3"
 BZIP2_VERSION="1.0.8"
 # We statically build the BSD libedit on selected platforms to get the
 # readline module available without linking to the GPL-only readline libs.
 LIBEDIT_VERSION="20170329-3.1"
-OPENSSL_VERSION="1.1.1t"
-SQLITE_VERSION="3.40.1"
+OPENSSL_VERSION="1.1.1w"
+SQLITE_VERSION="3.43.1"
 
 # Python modules versions to be used everywhere possible.
 PYSQLITE_VERSION="2.8.3"
 CFFI_VERSION="1.15.1"
 SCANDIR_VERSION="1.10.0"
-PSUTIL_VERSION="5.9.3"
+PSUTIL_VERSION="5.9.5"
 SUBPROCESS32_VERSION="3.5.4"
 
 # Versions no longer upgradable because of Python 2 deprecation.
@@ -35,14 +35,29 @@ PYOPENSSL_VERSION="21.0.0"
 # Backported fix for https://github.com/pypa/pip/issues/9827
 # at https://github.com/chevah/pip/tree/20.3.4chevah.
 PIP_VERSION="20.3.4chevah1"
-# For pip <21.1, click <8.0.0, dparse <0.5.2, wheel <0.38.0, safety <2.2.0,
-# setuptools <65.5.1, certifi <2022.12.07.
-SAFETY_IGNORED_OPTS="-i 40291 -i 47833 -i 50571 -i 51499 -i 51358 -i 52495 -i 52365"
+# For safety alerts, we need to ignore some vulnerabilities which are either:
+#   * not present in the final tarball, e.g. for wheel, safety, etc.,
+#   * not at all relevant, e.g. those for cryptography's bundled openssl,
+#   * not actually relevant for these old versions, e.g. 53048 for cryptography,
+#   * patched by us, e.g. 40291 for pip, 53048 for cryptography,
+#   * not patched: 52495 for setuptools.
+# pip <21.1, click <8, dparse <0.5.2, wheel <0.38, safety <2.2, pywin32 <301.
+SAFETY_IGNORED_OPTS="-i 40291 -i 47833 -i 50571 -i 51499 -i 51358 -i 54687"
+# setuptools <65.5.1, requests <2.31.0, certifi <2023.07.22.
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 52495 -i 58755 -i 52365 -i 59956"
+# These are related to cryptography's bundled OpenSSL libs. We don't use those.
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53306 -i 53298 -i 53305 -i 53301"
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53307 -i 53304 -i 53302 -i 53299"
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53303 -i 59062 -i 60225 -i 60223"
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 60224"
+# Other cryptography vulnerabilities, see a few lines above for more details.
+SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53048 -i 59473"
 # setuptools 44.x is the last series to support Python 2.7.
 # More at https://github.com/pypa/setuptools/pull/1955.
 SETUPTOOLS_VERSION="44.1.1"
-# Version 3.2.1 is used with OpenSSL 1.0.2 libs.
-CRYPTOGRAPHY_VERSION="3.3.2"
+# Version 3.2.1 (with patches) from python-modules/ is used with OpenSSL 1.0.2.
+# Our patched versions fix CVE-2023-23931.
+CRYPTOGRAPHY_VERSION="3.3.2chevah"
 # bcrypt 3.2.0 requires at least Python 3.6.
 BCRYPT_VERSION="3.1.7"
 # setproctitle 1.2.x requires at least Python 3.6.
@@ -164,7 +179,7 @@ case $OS in
         # MSYS2's Perl is not good enough for building OpenSSL.
         export PATH="/c/Strawberry/perl/bin/:$PATH:/c/Program Files/NASM/"
         export BUILD_OPENSSL="yes"
-        # Extra libraries are installed only using PIP.
+        # Python modules are installed only using PIP.
         EXTRA_LIBRARIES=""
         PIP_LIBRARIES="$PIP_LIBRARIES \
             pywin32==${PYWIN32_VERSION} \
@@ -202,8 +217,8 @@ case $OS in
         export BUILD_LIBEDIT="no"
         # As of January 2021, OpenSSL 1.0.2u is the latest version from IBM.
         export BUILD_OPENSSL="yes"
-        # 1.1.1 tests fail on AIX, use 1.0.2 with patches from Ubuntu 16.04 LTS.
-        OPENSSL_VERSION="1.0.2v-chevah4"
+        # 1.1.1 tests fail on AIX, use 1.0.2 with patches from CentOS 7.
+        OPENSSL_VERSION="1.0.2v-chevah5"
         # Perl's Test::Simple and its deps are required for building OpenSSL.
         execute perl -MTest::Simple -e 1
         # cryptography 3.2.x, last version to support OpenSSL 1.0.2.
@@ -236,9 +251,13 @@ case $OS in
         export BUILD_LIBFFI="yes"
         # OpenSSL 1.0.2 has extended support: https://tinyurl.com/2ck2sm6s.
         export CRYPTOGRAPHY_ALLOW_OPENSSL_102="yes"
-        # Put cryptography back, to build it against system OpenSSL.
+        # Build cryptography against system OpenSSL with our patches.
         # cryptography 3.2.1, last version working with OpenSSL 1.0.2.
-        PIP_LIBRARIES="cryptography==3.2.1 $PIP_LIBRARIES_OPENSSL_102"
+        EXTRA_LIBRARIES="$EXTRA_LIBRARIES \
+            python-modules/cryptography-3.2.1 \
+        "
+        # Use the appropriate PIP_LIBRARIES env var.
+        PIP_LIBRARIES="$PIP_LIBRARIES_OPENSSL_102"
         add_ignored_safety_ids_for_cryptography32
         ;;
     macos)
@@ -250,8 +269,12 @@ case $OS in
         # setup.py skips building readline by default, as it sets this to
         # "10.4", and then tries to avoid the broken readline in OS X 10.4.
         export MACOSX_DEPLOYMENT_TARGET=10.12
-        # System includes bzip2 libs by default.
-        export BUILD_BZIP2="no"
+        # System included bzip2 libs by default up to and including macOS 10.15.
+        export BUILD_BZIP2="yes"
+        # Apparently, macOS 11 doesn't include zlib libraries either.
+        export BUILD_ZLIB="yes"
+        # Building readline fails on macOS 11, didn't look into it.
+        export BUILD_LIBEDIT="no"
         # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs.
         # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7.
         # Therefore, we build OpenSSL for both stdlib and cryptography.
@@ -302,18 +325,6 @@ case $OS in
         export PATH="/usr/local/bin:$PATH"
         # In particular, Perl's Test::Simple and its deps are required.
         execute perl -MTest::Simple -e 1
-        # Version 5.9.2-5.9.4 of psutil not working properly on CentOS 5.
-        # More at https://github.com/giampaolo/psutil/issues/2164.
-        # Should be fixed with the 5.9.5 release.
-        PIP_LIBRARIES="\
-            cryptography==${CRYPTOGRAPHY_VERSION} \
-            pyOpenSSL==${PYOPENSSL_VERSION} \
-            scandir==${SCANDIR_VERSION} \
-            subprocess32==${SUBPROCESS32_VERSION} \
-            bcrypt==${BCRYPT_VERSION} \
-            psutil=="5.9.1" \
-            setproctitle==${SETPROCTITLE_VERSION}
-            "
         ;;
     *)
         # Only supported Linux distributions should be left.
@@ -706,6 +717,10 @@ command_test() {
 
     echo '##### Testing for outdated packages and security issues... #####'
     execute $PYTHON_BIN -m pip list --outdated --format=columns
+    # Install wheel back for better collection of needed dependencies.
+    execute $PYTHON_BIN -m pip install $PIP_ARGS wheel
+    # Move include/ back for building some deps, like Cython on ARM64.
+    execute mv $INSTALL_FOLDER/lib/include $INSTALL_FOLDER/
     # Safety needs PyYAML, which needs Cython, which needs to be built on AIX.
     aix_ld_hack init
     # This is the newest version that still works with Python 2.7.x.
@@ -742,8 +757,8 @@ command_test() {
             (>&2 echo -e "\tSkipping because of upstream issues.")
             ;;
         lnx*)
-            if [ x${CHEVAH_CONTAINER-} = x"yes" ]; then
-                (>&2 echo -e "\tSkipping as it fails under Docker on CentOS 5.")
+            if [ -f /.dockerenv ]; then
+                (>&2 echo -e "\tSkipping as it fails under Docker.")
             else
                 execute $PYTHON_BIN ${SCANDIR_FOLDER}/test/run_tests.py
             fi
@@ -771,7 +786,7 @@ command_compat() {
     execute pushd build
     echo '##### Running chevah.compat tests... #####'
     execute rm -rf compat
-    execute git clone https://github.com/chevah/compat.git --depth=1 -b master
+    execute git clone https://github.com/chevah/compat.git --depth=1 -b py2-support
     execute pushd compat
     # Copy over current brink stuff, as some changes might require it.
     execute cp ../../brink.{conf,sh} ./
@@ -784,8 +799,6 @@ command_compat() {
     execute cp -r ../$LOCAL_PYTHON_BINARY_DIST cache/
     # Make sure everything is done from scratch in the current dir.
     unset CHEVAH_CACHE CHEVAH_BUILD
-    # Install wheel back for the compat tests.
-    execute $PYTHON_BIN -m pip install $PIP_ARGS wheel
     # Some tests might fail due to causes which are not related to python.
     execute ./brink.sh deps
     if [ "${CHEVAH_CONTAINER:-}" = "yes" ]; then