pythongh-124064: Make warning emitting compiler options opt-in (python#124070)

Co-authored-by: Hugo van Kemenade <[email protected]>
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/reusable-macos.yml

@@ -35,7 +35,7 @@ jobs:
         path: config.cache
         key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
-      run: brew install pkg-config [email protected] xz gdbm tcl-tk
+      run: brew install pkg-config [email protected] xz gdbm tcl-tk make
     - name: Configure CPython
       run: |
         GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -44,14 +44,27 @@ jobs:
           --config-cache \
           --with-pydebug \
           --enable-slower-safety \
+          --enable-safety \
           ${{ inputs.free-threading && '--disable-gil' || '' }} \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
-      run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+      if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+      run: gmake -j8
+    - name: Build CPython for compiler warning check
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
     - name: Display build info
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: >-
+        python3 Tools/build/check_warnings.py
+        --compiler-output-file-path=compiler_output_macos.txt
+        --warning-ignore-file-path=Tools/build/.warningignore_macos
+        --compiler-output-type=clang
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="./"
     - name: Tests
       run: make test

.github/workflows/reusable-ubuntu.yml

@@ -67,20 +67,33 @@ jobs:
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: >-
         ../cpython-ro-srcdir/configure
-        CFLAGS="-fdiagnostics-format=json"
         --config-cache
         --with-pydebug
         --enable-slower-safety
+        --enable-safety
         --with-openssl=$OPENSSL_DIR
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Build CPython out-of-tree
+      if: ${{ inputs.free-threading }}
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+      run: make -j4
+    - name: Build CPython out-of-tree (for compiler warning check)
+      if: ${{ !inputs.free-threading}}
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt
     - name: Display build info
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+      if: ${{ !inputs.free-threading }}
+      run: >-
+        python Tools/build/check_warnings.py
+        --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+        --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+        --compiler-output-type=gcc
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="../cpython-ro-srcdir/"
     - name: Remount sources writable for tests
       # some tests write to srcdir, lack of pyc files slows down testing
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

Misc/NEWS.d/next/Security/2024-08-14-19-43-57.gh-issue-112301.IQUcOy.rst

@@ -0,0 +1 @@
+Enable compiler options that warn of potential security vulnerabilities.

Tools/build/.warningignore_macos

@@ -3,3 +3,230 @@
 # Keep lines sorted lexicographically to help avoid merge conflicts.
 # Format example:
 # /path/to/file (number of warnings in file)
+Include/internal/mimalloc/mimalloc/internal.h 4
+Include/internal/pycore_backoff.h 1
+Include/internal/pycore_dict.h 2
+Include/internal/pycore_gc.h 1
+Include/internal/pycore_long.h 2
+Include/internal/pycore_object.h 4
+Modules/_asynciomodule.c 3
+Modules/_bisectmodule.c 2
+Modules/_bz2module.c 5
+Modules/_collectionsmodule.c 2
+Modules/_csv.c 3
+Modules/_ctypes/_ctypes.c 37
+Modules/_ctypes/_ctypes_test_generated.c.h 141
+Modules/_ctypes/callbacks.c 6
+Modules/_ctypes/callproc.c 15
+Modules/_ctypes/cfield.c 56
+Modules/_ctypes/malloc_closure.c 3
+Modules/_ctypes/stgdict.c 17
+Modules/_cursesmodule.c 24
+Modules/_datetimemodule.c 28
+Modules/_dbmmodule.c 8
+Modules/_decimal/_decimal.c 15
+Modules/_elementtree.c 42
+Modules/_functoolsmodule.c 6
+Modules/_gdbmmodule.c 5
+Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c 84
+Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c 84
+Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h 24
+Modules/_hashopenssl.c 16
+Modules/_interpchannelsmodule.c 1
+Modules/_interpqueuesmodule.c 1
+Modules/_io/_iomodule.c 1
+Modules/_io/bufferedio.c 4
+Modules/_io/bytesio.c 11
+Modules/_io/fileio.c 9
+Modules/_io/stringio.c 8
+Modules/_io/textio.c 11
+Modules/_json.c 19
+Modules/_localemodule.c 3
+Modules/_lzmamodule.c 10
+Modules/_multiprocessing/semaphore.c 2
+Modules/_operator.c 5
+Modules/_pickle.c 71
+Modules/_posixsubprocess.c 8
+Modules/_queuemodule.c 4
+Modules/_randommodule.c 3
+Modules/_scproxy.c 3
+Modules/_sqlite/connection.c 4
+Modules/_sqlite/cursor.c 3
+Modules/_sqlite/module.c 2
+Modules/_sre/sre.c 18
+Modules/_sre/sre_lib.h 62
+Modules/_ssl.c 29
+Modules/_struct.c 1
+Modules/_testbuffer.c 22
+Modules/_testcapi/heaptype.c 1
+Modules/_testcapi/long.c 2
+Modules/_testcapi/mem.c 2
+Modules/_testcapi/monitoring.c 3
+Modules/_testcapi/pyatomic.c 1
+Modules/_testcapi/unicode.c 2
+Modules/_testcapi/vectorcall.c 3
+Modules/_testcapi/watchers.c 3
+Modules/_testcapimodule.c 3
+Modules/_testclinic.c 14
+Modules/_testexternalinspection.c 8
+Modules/_testinternalcapi.c 8
+Modules/_testinternalcapi/pytime.c 8
+Modules/_testinternalcapi/test_critical_sections.c 1
+Modules/_testinternalcapi/test_lock.c 2
+Modules/_testlimitedcapi/heaptype_relative.c 4
+Modules/_testlimitedcapi/object.c 2
+Modules/_testlimitedcapi/unicode.c 2
+Modules/_threadmodule.c 2
+Modules/_tkinter.c 6
+Modules/_xxtestfuzz/_xxtestfuzz.c 1
+Modules/_xxtestfuzz/fuzzer.c 11
+Modules/_zoneinfo.c 14
+Modules/arraymodule.c 32
+Modules/atexitmodule.c 1
+Modules/binascii.c 206
+Modules/blake2module.c 6
+Modules/cjkcodecs/_codecs_cn.c 1
+Modules/cjkcodecs/_codecs_iso2022.c 2
+Modules/cjkcodecs/_codecs_jp.c 14
+Modules/cjkcodecs/_codecs_kr.c 3
+Modules/cjkcodecs/cjkcodecs.h 1
+Modules/cjkcodecs/multibytecodec.c 2
+Modules/clinic/_testclinic.c.h 1
+Modules/clinic/arraymodule.c.h 1
+Modules/clinic/unicodedata.c.h 10
+Modules/cmathmodule.c 1
+Modules/expat/siphash.h 8
+Modules/expat/xmlparse.c 45
+Modules/expat/xmltok.c 17
+Modules/expat/xmltok_impl.c 34
+Modules/faulthandler.c 3
+Modules/fcntlmodule.c 1
+Modules/getpath.c 7
+Modules/grpmodule.c 4
+Modules/itertoolsmodule.c 7
+Modules/main.c 2
+Modules/mathmodule.c 15
+Modules/mmapmodule.c 20
+Modules/posixmodule.c 67
+Modules/pwdmodule.c 4
+Modules/pyexpat.c 20
+Modules/readline.c 1
+Modules/resource.c 3
+Modules/rotatingtree.c 1
+Modules/selectmodule.c 6
+Modules/sha3module.c 4
+Modules/signalmodule.c 1
+Modules/socketmodule.c 44
+Modules/syslogmodule.c 3
+Modules/timemodule.c 4
+Modules/unicodedata.c 28
+Modules/unicodedata_db.h 1
+Modules/xxsubtype.c 2
+Modules/zlibmodule.c 16
+Objects/abstract.c 2
+Objects/bytearrayobject.c 34
+Objects/bytes_methods.c 9
+Objects/bytesobject.c 35
+Objects/call.c 13
+Objects/classobject.c 4
+Objects/codeobject.c 15
+Objects/descrobject.c 2
+Objects/dictobject.c 28
+Objects/fileobject.c 3
+Objects/floatobject.c 30
+Objects/frameobject.c 19
+Objects/funcobject.c 1
+Objects/genobject.c 5
+Objects/listobject.c 43
+Objects/longobject.c 46
+Objects/memoryobject.c 6
+Objects/methodobject.c 1
+Objects/mimalloc/alloc.c 6
+Objects/mimalloc/arena.c 6
+Objects/mimalloc/heap.c 1
+Objects/mimalloc/init.c 2
+Objects/mimalloc/options.c 1
+Objects/mimalloc/os.c 4
+Objects/mimalloc/page-queue.c 2
+Objects/mimalloc/page.c 1
+Objects/mimalloc/prim/osx/../unix/prim.c 2
+Objects/mimalloc/random.c 1
+Objects/mimalloc/segment.c 11
+Objects/mimalloc/stats.c 1
+Objects/moduleobject.c 2
+Objects/object.c 1
+Objects/obmalloc.c 6
+Objects/odictobject.c 3
+Objects/rangeobject.c 10
+Objects/setobject.c 13
+Objects/sliceobject.c 4
+Objects/stringlib/codecs.h 26
+Objects/stringlib/eq.h 1
+Objects/stringlib/fastsearch.h 14
+Objects/stringlib/join.h 1
+Objects/stringlib/replace.h 4
+Objects/stringlib/repr.h 21
+Objects/stringlib/transmogrify.h 5
+Objects/structseq.c 14
+Objects/tupleobject.c 10
+Objects/typeobject.c 17
+Objects/unicodectype.c 7
+Objects/unicodeobject.c 113
+Parser/action_helpers.c 4
+Parser/lexer/buffer.c 1
+Parser/lexer/lexer.c 12
+Parser/parser.c 116
+Parser/pegen.c 7
+Parser/string_parser.c 7
+Parser/tokenizer/file_tokenizer.c 8
+Parser/tokenizer/helpers.c 7
+Parser/tokenizer/readline_tokenizer.c 3
+Programs/_freeze_module.c 1
+Python/Python-ast.c 15
+Python/asdl.c 3
+Python/assemble.c 7
+Python/ast_opt.c 7
+Python/bltinmodule.c 9
+Python/bootstrap_hash.c 4
+Python/ceval.c 8
+Python/ceval_gil.c 2
+Python/codecs.c 32
+Python/codegen.c 6
+Python/compile.c 2
+Python/context.c 1
+Python/crossinterp.c 2
+Python/crossinterp_data_lookup.h 1
+Python/dtoa.c 34
+Python/errors.c 1
+Python/fileutils.c 7
+Python/flowgraph.c 8
+Python/formatter_unicode.c 7
+Python/frame.c 4
+Python/gc.c 8
+Python/generated_cases.c.h 35
+Python/getargs.c 11
+Python/import.c 5
+Python/initconfig.c 11
+Python/instrumentation.c 31
+Python/intrinsics.c 1
+Python/legacy_tracing.c 3
+Python/lock.c 4
+Python/marshal.c 11
+Python/modsupport.c 3
+Python/mystrtoul.c 4
+Python/pathconfig.c 1
+Python/preconfig.c 2
+Python/pyarena.c 1
+Python/pyhash.c 2
+Python/pylifecycle.c 7
+Python/pystate.c 6
+Python/pystrhex.c 19
+Python/pystrtod.c 3
+Python/qsbr.c 2
+Python/specialize.c 10
+Python/suggestions.c 12
+Python/symtable.c 18
+Python/sysmodule.c 2
+Python/thread_pthread.h 1
+Python/traceback.c 6
+Python/tracemalloc.c 6