diff --git a/.cirrus.yml b/.cirrus.yml index f902512c65..f059d2ea38 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,33 +1,32 @@ -# TODO(iphydf): Reactivate when we have quota again in February 2025. -# --- -# freebsd_task: -# timeout_in: 5m -# freebsd_instance: -# image_family: freebsd-14-1 -# configure_script: -# - PAGER=cat ASSUME_ALWAYS_YES=YES pkg install -# cmake -# git -# gmake -# googletest -# libconfig -# libsodium -# libvpx -# ninja -# opus -# pkgconf -# - git submodule update --init --recursive -# test_all_script: -# - | -# # TODO(iphydf): Investigate FreeBSD failures on these tests. -# sed -Ei -e '/\(dht_nodes_request_api\)/s/^/#/' auto_tests/CMakeLists.txt -# cmake . \ -# -DMIN_LOGGER_LEVEL=TRACE \ -# -DMUST_BUILD_TOXAV=ON \ -# -DNON_HERMETIC_TESTS=OFF \ -# -DTEST_TIMEOUT_SECONDS=50 \ -# -DUSE_IPV6=OFF \ -# -DAUTOTEST=ON \ -# -GNinja -# cmake --build . --target install -# ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:3 || ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:3 +--- +freebsd_task: + timeout_in: 5m + freebsd_instance: + image_family: freebsd-14-1 + configure_script: + - PAGER=cat ASSUME_ALWAYS_YES=YES pkg install + cmake + git + gmake + googletest + libconfig + libsodium + libvpx + ninja + opus + pkgconf + - git submodule update --init --recursive + test_all_script: + - | + # TODO(iphydf): Investigate FreeBSD failures on these tests. + sed -Ei -e '/\(dht_nodes_request_api\)/s/^/#/' auto_tests/CMakeLists.txt + cmake . \ + -DMIN_LOGGER_LEVEL=TRACE \ + -DMUST_BUILD_TOXAV=ON \ + -DNON_HERMETIC_TESTS=OFF \ + -DTEST_TIMEOUT_SECONDS=50 \ + -DUSE_IPV6=OFF \ + -DAUTOTEST=ON \ + -GNinja + cmake --build . --target install + ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:3 || ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:3 diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile index e563011f5a..4fa7632d64 100644 --- a/.clusterfuzzlite/Dockerfile +++ b/.clusterfuzzlite/Dockerfile @@ -3,19 +3,23 @@ # We want to use the latest tools always FROM gcr.io/oss-fuzz-base/base-builder:latest -RUN apt-get update && \ - apt-get -y install --no-install-suggests --no-install-recommends \ - cmake libtool autoconf automake pkg-config \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get -y install --no-install-suggests --no-install-recommends \ + cmake \ + pkg-config \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Static builds of dependencies # libsodium -RUN git clone --depth 1 --branch 1.0.18 https://github.com/jedisct1/libsodium libsodium -WORKDIR $SRC/libsodium -RUN ./autogen.sh && ./configure --enable-shared=no && make install -WORKDIR $SRC +RUN tar zxf <(curl -L https://github.com/jedisct1/libsodium/releases/download/1.0.20-RELEASE/libsodium-1.0.20.tar.gz) \ + && cd libsodium-* \ + && ./configure --enable-shared=no \ + && make install \ + && cd .. # Copy your project's source code. COPY . $SRC/c-toxcore diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh index 59adb615d6..9496f15dcf 100644 --- a/.clusterfuzzlite/build.sh +++ b/.clusterfuzzlite/build.sh @@ -1,6 +1,16 @@ #!/bin/bash -eu -FUZZ_TARGETS="bootstrap_fuzz_test toxsave_fuzz_test" +FUZZ_TARGETS=( + DHT_fuzz_test + bootstrap_fuzz_test + # e2e_fuzz_test + forwarding_fuzz_test + group_announce_fuzz_test + group_moderation_fuzz_test + net_crypto_fuzz_test + tox_events_fuzz_test + toxsave_fuzz_test +) # out of tree build cd "$WORK" @@ -12,11 +22,10 @@ cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER="$CC" \ -DCMAKE_CXX_COMPILER="$CXX" \ -DCMAKE_C_FLAGS="$CFLAGS" \ -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ - -DCMAKE_EXE_LINKER_FLAGS="$LIB_FUZZING_ENGINE" \ -DBUILD_TOXAV=OFF -DENABLE_SHARED=NO -DBUILD_FUZZ_TESTS=ON \ -DDHT_BOOTSTRAP=OFF -DBOOTSTRAP_DAEMON=OFF "$SRC"/c-toxcore -for TARGET in $FUZZ_TARGETS; do +for TARGET in "${FUZZ_TARGETS[@]}"; do # build fuzzer target cmake --build ./ --target "$TARGET" diff --git a/.github/scripts/flags-clang.sh b/.github/scripts/flags-clang.sh index f0d9a11c87..02fbd766e4 100644 --- a/.github/scripts/flags-clang.sh +++ b/.github/scripts/flags-clang.sh @@ -69,6 +69,8 @@ add_cxx_flag -Wno-c99-extensions add_cxx_flag -Wno-old-style-cast # GTest does this. add_cxx_flag -Wno-global-constructors +# Needed for some fuzzers. +add_cxx_flag -Wno-exit-time-destructors # Downgrade to warning so we still see it. add_flag -Wno-error=unreachable-code diff --git a/.github/workflows/cflite_batch.yml b/.github/workflows/cflite_batch.yml index d853507928..2a018c271c 100644 --- a/.github/workflows/cflite_batch.yml +++ b/.github/workflows/cflite_batch.yml @@ -1,11 +1,14 @@ # Derived from: https://google.github.io/clusterfuzzlite/running-clusterfuzzlite/github-actions/ name: ClusterFuzzLite batch fuzzing + on: schedule: - cron: '0 6,8 * * *' # Run twice a day at low activity times workflow_dispatch: # Manual trigger for testing + permissions: read-all + jobs: BatchFuzzing: runs-on: ubuntu-latest diff --git a/.github/workflows/cflite_cron.yml b/.github/workflows/cflite_cron.yml index c127a67a33..c544a6df5d 100644 --- a/.github/workflows/cflite_cron.yml +++ b/.github/workflows/cflite_cron.yml @@ -7,6 +7,7 @@ on: workflow_dispatch: # Manual trigger for testing permissions: read-all + jobs: Pruning: runs-on: ubuntu-latest @@ -26,6 +27,7 @@ jobs: storage-repo: https://${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com/TokTok/toktok-fuzzer.git storage-repo-branch: master # Optional. Defaults to "main" storage-repo-branch-coverage: gh-pages # Optional. Defaults to "gh-pages". + Coverage: runs-on: ubuntu-latest steps: @@ -47,4 +49,3 @@ jobs: storage-repo: https://${{ secrets.PERSONAL_ACCESS_TOKEN }}@github.com/TokTok/toktok-fuzzer.git storage-repo-branch: master # Optional. Defaults to "main" storage-repo-branch-coverage: gh-pages # Optional. Defaults to "gh-pages". - diff --git a/.github/workflows/cflite_pr.yml b/.github/workflows/cflite_pr.yml new file mode 100644 index 0000000000..f61644a864 --- /dev/null +++ b/.github/workflows/cflite_pr.yml @@ -0,0 +1,36 @@ +# Derived from: https://google.github.io/clusterfuzzlite/running-clusterfuzzlite/github-actions/ + +name: ClusterFuzzLite pull request fuzzing + +on: + pull_request: + types: [opened, synchronize] + +jobs: + Fuzzing: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + sanitizer: + - address + - undefined + - memory + steps: + - name: Build Fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@v1 + with: + sanitizer: ${{ matrix.sanitizer }} + - name: Run Fuzzers (${{ matrix.sanitizer }}) + id: run + uses: google/clusterfuzzlite/actions/run_fuzzers@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: 300 # 5 mins (total time, not per test) + mode: 'code-change' + sanitizer: ${{ matrix.sanitizer }} + # Optional but recommended: For storing certain artifacts from fuzzing. + # See later section on "Git repo for storage". + storage-repo: https://github.com/TokTok/toktok-fuzzer.git + storage-repo-branch: master # Optional. Defaults to "main" diff --git a/testing/fuzzing/CMakeLists.txt b/testing/fuzzing/CMakeLists.txt index e306112ff2..32a31097a9 100644 --- a/testing/fuzzing/CMakeLists.txt +++ b/testing/fuzzing/CMakeLists.txt @@ -20,7 +20,9 @@ function(fuzz_test target source_dir) endfunction() fuzz_test(bootstrap .) # Fuzzes the bootstrap process -fuzz_test(toxsave .) # Fuzzes the bootstrap process +# TODO(iphydf): Fix this in the cmake build. +# fuzz_test(e2e .) # Fuzzes an end-to-end connection +fuzz_test(toxsave .) # Fuzzes tox_new and tox_get_savedata fuzz_test(DHT ../../toxcore) fuzz_test(forwarding ../../toxcore) diff --git a/testing/fuzzing/toxsave_fuzz_test.cc b/testing/fuzzing/toxsave_fuzz_test.cc index f48c2bf013..0ba9737f9e 100644 --- a/testing/fuzzing/toxsave_fuzz_test.cc +++ b/testing/fuzzing/toxsave_fuzz_test.cc @@ -20,6 +20,8 @@ void TestSaveDataLoading(Fuzz_Data &input) const size_t savedata_size = input.size(); CONSUME_OR_RETURN(const uint8_t *savedata, input, savedata_size); + tox_options_set_experimental_groups_persistence(tox_options, true); + // pass test data to Tox tox_options_set_savedata_data(tox_options, savedata, savedata_size); tox_options_set_savedata_type(tox_options, TOX_SAVEDATA_TYPE_TOX_SAVE); diff --git a/toxcore/forwarding_fuzz_test.cc b/toxcore/forwarding_fuzz_test.cc index 4b8521ab07..829d6947c8 100644 --- a/toxcore/forwarding_fuzz_test.cc +++ b/toxcore/forwarding_fuzz_test.cc @@ -13,11 +13,11 @@ namespace { std::optional> prepare(Fuzz_Data &input) { CONSUME_OR_RETURN_VAL(const uint8_t *ipp_packed, input, SIZE_IP_PORT, std::nullopt); - IP_Port ipp; + IP_Port ipp{}; unpack_ip_port(&ipp, ipp_packed, SIZE_IP6, true); CONSUME_OR_RETURN_VAL(const uint8_t *forwarder_packed, input, SIZE_IP_PORT, std::nullopt); - IP_Port forwarder; + IP_Port forwarder{}; unpack_ip_port(&forwarder, forwarder_packed, SIZE_IP6, true); // 2 bytes: size of the request @@ -37,22 +37,22 @@ void TestSendForwardRequest(Fuzz_Data &input) const uint16_t chain_keys_size = chain_length * CRYPTO_PUBLIC_KEY_SIZE; CONSUME_OR_RETURN(const uint8_t *chain_keys, input, chain_keys_size); - auto prep = prepare(input); + const auto prep = prepare(input); if (!prep.has_value()) { return; } - auto [ipp, forwarder, data, data_size] = prep.value(); + const auto [ipp, forwarder, data, data_size] = prep.value(); // rest of the fuzz data is input for malloc and network Fuzz_System sys(input); - Ptr logger(logger_new(sys.mem.get()), logger_kill); + const Ptr logger(logger_new(sys.mem.get()), logger_kill); if (logger == nullptr) { return; } - Ptr net(new_networking_ex(logger.get(), sys.mem.get(), sys.ns.get(), &ipp.ip, - ipp.port, ipp.port + 100, nullptr), + const Ptr net(new_networking_ex(logger.get(), sys.mem.get(), sys.ns.get(), + &ipp.ip, ipp.port, ipp.port + 100, nullptr), kill_networking); if (net == nullptr) { return; @@ -66,22 +66,22 @@ void TestForwardReply(Fuzz_Data &input) CONSUME1_OR_RETURN(const uint16_t, sendback_length, input); CONSUME_OR_RETURN(const uint8_t *sendback, input, sendback_length); - auto prep = prepare(input); + const auto prep = prepare(input); if (!prep.has_value()) { return; } - auto [ipp, forwarder, data, data_size] = prep.value(); + const auto [ipp, forwarder, data, data_size] = prep.value(); // rest of the fuzz data is input for malloc and network Fuzz_System sys(input); - Ptr logger(logger_new(sys.mem.get()), logger_kill); + const Ptr logger(logger_new(sys.mem.get()), logger_kill); if (logger == nullptr) { return; } - Ptr net(new_networking_ex(logger.get(), sys.mem.get(), sys.ns.get(), &ipp.ip, - ipp.port, ipp.port + 100, nullptr), + const Ptr net(new_networking_ex(logger.get(), sys.mem.get(), sys.ns.get(), + &ipp.ip, ipp.port, ipp.port + 100, nullptr), kill_networking); if (net == nullptr) { return;