diff --git a/.circleci/compare_bytecode_reports.sh b/.circleci/compare_bytecode_reports.sh index 7b62656bf..7574b0656 100755 --- a/.circleci/compare_bytecode_reports.sh +++ b/.circleci/compare_bytecode_reports.sh @@ -30,6 +30,7 @@ native_platforms=( ubuntu2004-static ubuntu osx + osx_intel windows ) interfaces=( diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a967feba..02c15e691 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,6 +81,7 @@ commands: enum: - solcjs - native + - osx_intel binary_path: type: string preset: @@ -491,12 +492,13 @@ defaults: - base_osx: &base_osx macos: - xcode: "14.2.0" - resource_class: macos.x86.medium.gen2 + xcode: "15.0.0" + resource_class: macos.m1.medium.gen1 environment: &base_osx_env TERM: xterm MAKEFLAGS: -j5 CPUs: 5 + ETH_EVMONE: /usr/local/lib/libevmone.dylib - base_python_small: &base_python_small docker: @@ -1146,6 +1148,7 @@ jobs: environment: <<: *base_osx_env CMAKE_BUILD_TYPE: Release + CMAKE_OPTIONS: -DCMAKE_OSX_ARCHITECTURES:STRING=x86_64;arm64 steps: - checkout - install_dependencies_osx @@ -1657,6 +1660,22 @@ jobs: binary_path: "build/solc/solc" preset: "<< parameters.preset >>" + b_bytecode_osx_intel: + parameters: + preset: + type: string + <<: *base_osx + parallelism: 2 # For prepare_bytecode_report + steps: + - checkout + - attach_workspace: + at: . + - prepare_bytecode_report: + label: "osx_intel" + binary_type: osx_intel + binary_path: "build/solc/solc" + preset: "<< parameters.preset >>" + b_bytecode_win: parameters: preset: @@ -1880,6 +1899,11 @@ workflows: matrix: *bytecode_compare_preset_matrix requires: - b_osx + - b_bytecode_osx_intel: + <<: *on_all_tags_and_branches + matrix: *bytecode_compare_preset_matrix + requires: + - b_osx - b_bytecode_ems: <<: *on_all_tags_and_branches matrix: *bytecode_compare_preset_matrix @@ -1892,6 +1916,7 @@ workflows: - b_bytecode_ubu - b_bytecode_win - b_bytecode_osx + - b_bytecode_osx_intel - b_bytecode_ems # Final artifacts diff --git a/.circleci/osx_install_dependencies.sh b/.circleci/osx_install_dependencies.sh index 8b8656abd..4ca4a31d5 100755 --- a/.circleci/osx_install_dependencies.sh +++ b/.circleci/osx_install_dependencies.sh @@ -50,33 +50,67 @@ if [ ! -f /usr/local/lib/libz3.a ] # if this file does not exists (cache was not then brew update brew upgrade - brew install boost brew install cmake brew install wget brew install coreutils brew install diffutils brew install grep - ./scripts/install_obsolete_jsoncpp_1_7_4.sh + + # writing to /usr/local/lib need administrative privileges. + sudo ./scripts/install_obsolete_jsoncpp_1_7_4.sh + + # boost + boost_version="1.84.0" + boost_package="boost_${boost_version//./_}.tar.bz2" + boost_dir="boost_${boost_version//./_}" + wget "https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source/$boost_package" + tar xf "$boost_package" + rm "$boost_package" + cd "$boost_dir" + ./bootstrap.sh --with-toolset=clang --with-libraries=thread,system,filesystem,program_options,serialization,test + # the default number of jobs that b2 is taking, is the number of detected available CPU threads. + sudo ./b2 -a address-model=64 architecture=arm+x86 install + cd .. + sudo rm -rf "$boost_dir" # z3 z3_version="4.12.1" - z3_dir="z3-${z3_version}-x64-osx-10.16" - z3_package="${z3_dir}.zip" - wget "https://github.com/Z3Prover/z3/releases/download/z3-${z3_version}/${z3_package}" - validate_checksum "$z3_package" 7601f844de6d906235140d0f76cca58be7ac716f3e2c29c35845aa24b24f73b9 - unzip "$z3_package" + z3_dir="z3-z3-$z3_version" + z3_package="z3-$z3_version.tar.gz" + wget "https://github.com/Z3Prover/z3/archive/refs/tags/$z3_package" + validate_checksum "$z3_package" a3735fabf00e1341adcc70394993c05fd3e2ae167a3e9bb46045e33084eb64a3 + tar xf "$z3_package" rm "$z3_package" - cp "${z3_dir}/bin/libz3.a" /usr/local/lib - cp "${z3_dir}/bin/z3" /usr/local/bin - cp "${z3_dir}/include/"* /usr/local/include - rm -r "$z3_dir" + cd "$z3_dir" + mkdir build + cd build + cmake -DCMAKE_OSX_ARCHITECTURES:STRING="x86_64;arm64" -DZ3_BUILD_LIBZ3_SHARED=false .. + make -j + sudo make install + cd ../.. + rm -rf "$z3_dir" # evmone evmone_version="0.11.0" - evmone_package="evmone-${evmone_version}-darwin-x86_64.tar.gz" - wget "https://github.com/ethereum/evmone/releases/download/v${evmone_version}/${evmone_package}" - validate_checksum "$evmone_package" 83ed20676681d9a31bd30cac399ab7c615ccab8adb8087cc2c7e9cd22b4d2efc - tar xzpf "$evmone_package" -C /usr/local - rm "$evmone_package" - + if [[ $(uname -m) == 'arm64' ]] + then + # evmone does not provide any builds for apple silicon yet. so lets just build it locally. + # be aware that we are only building the arm version here, we don't build a universal binary. + git clone https://github.com/ethereum/evmone.git + cd evmone + git checkout "v${evmone_version}" + git submodule update --init + cmake -S . -B build + cmake --build build + cd build + sudo make install + cd ../.. + rm -rf evmone + else + evmone_package="evmone-${evmone_version}-darwin-x86_64.tar.gz" + wget "https://github.com/ethereum/evmone/releases/download/v${evmone_version}/${evmone_package}" + validate_checksum "$evmone_package" 83ed20676681d9a31bd30cac399ab7c615ccab8adb8087cc2c7e9cd22b4d2efc + tar xzpf "$evmone_package" -C /usr/local + rm "$evmone_package" + fi fi diff --git a/.circleci/parallel_bytecode_report.sh b/.circleci/parallel_bytecode_report.sh index 9b316266a..7fe22c343 100755 --- a/.circleci/parallel_bytecode_report.sh +++ b/.circleci/parallel_bytecode_report.sh @@ -33,7 +33,7 @@ binary_type="$2" binary_path="$3" # This path must be absolute preset="$4" -[[ $binary_type == native || $binary_type == solcjs ]] || { >&2 echo "Invalid binary type: ${binary_type}"; exit 1; } +[[ $binary_type == native || $binary_type == "osx_intel" || $binary_type == solcjs ]] || { >&2 echo "Invalid binary type: ${binary_type}"; exit 1; } # NOTE: Locale affects the order of the globbed files. export LC_ALL=C @@ -47,7 +47,7 @@ python3 ../scripts/isolate_tests.py ../test/ # FIXME: These cases crash because of https://github.com/ethereum/solidity/issues/13583 rm ./*_bytecode_too_large_*.sol ./*_combined_too_large_*.sol -if [[ $binary_type == native ]]; then +if [[ $binary_type == native || $binary_type == "osx_intel" ]]; then interface=$(echo -e "standard-json\ncli" | circleci tests split) echo "Selected interface: ${interface}" @@ -56,6 +56,7 @@ if [[ $binary_type == native ]]; then "$binary_path" \ --interface "$interface" \ --preset "$preset" \ + --execution-arch "$binary_type" \ --report-file "../bytecode-report-${label}-${interface}-${preset}.txt" else echo "Installing solc-js" diff --git a/cmake/jsoncpp.cmake b/cmake/jsoncpp.cmake index cf1cc656b..c47c2697a 100644 --- a/cmake/jsoncpp.cmake +++ b/cmake/jsoncpp.cmake @@ -40,6 +40,7 @@ if (WIN32 AND POLICY CMP0091 AND CMAKE_MSVC_RUNTIME_LIBRARY) list(APPEND JSONCPP_CMAKE_ARGS "-DCMAKE_MSVC_RUNTIME_LIBRARY=${CMAKE_MSVC_RUNTIME_LIBRARY}") endif() +string(REPLACE ";" "$" CMAKE_OSX_ARCHITECTURES_ "${CMAKE_OSX_ARCHITECTURES}") ExternalProject_Add(jsoncpp-project PREFIX "${prefix}" DOWNLOAD_DIR "${PROJECT_SOURCE_DIR}/deps/downloads" @@ -57,6 +58,7 @@ ExternalProject_Add(jsoncpp-project -DJSONCPP_WITH_PKGCONFIG_SUPPORT=OFF -DCMAKE_CXX_FLAGS=${JSONCPP_CXX_FLAGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES_} ${JSONCPP_CMAKE_ARGS} ${byproducts} ) diff --git a/scripts/bytecodecompare/prepare_report.py b/scripts/bytecodecompare/prepare_report.py index 8be9ae045..938f5171f 100755 --- a/scripts/bytecodecompare/prepare_report.py +++ b/scripts/bytecodecompare/prepare_report.py @@ -26,6 +26,11 @@ class CompilerInterface(Enum): STANDARD_JSON = 'standard-json' +class ExecutionArchitecture(Enum): + NATIVE = 'native' + OSX_INTEL = 'osx_intel' + + class SettingsPreset(Enum): LEGACY_OPTIMIZE = 'legacy-optimize' LEGACY_NO_OPTIMIZE = 'legacy-no-optimize' @@ -214,6 +219,7 @@ def parse_cli_output(source_file_name: Path, cli_output: str, exit_code: int) -> def prepare_compiler_input( compiler_path: Path, + execution_arch: ExecutionArchitecture, source_file_name: Path, force_no_optimize_yul: bool, interface: CompilerInterface, @@ -224,6 +230,12 @@ def prepare_compiler_input( settings = CompilerSettings.from_preset(preset) + command_line = [] + if execution_arch == ExecutionArchitecture.OSX_INTEL: + command_line = ["/usr/bin/arch", "-64", "-x86_64"] + else: + assert execution_arch == ExecutionArchitecture.NATIVE + if interface == CompilerInterface.STANDARD_JSON: json_input: dict = { 'language': 'Solidity', @@ -241,7 +253,7 @@ def prepare_compiler_input( if smt_use == SMTUse.DISABLE: json_input['settings']['modelChecker'] = {'engine': 'none'} - command_line = [str(compiler_path), '--standard-json'] + command_line += [str(compiler_path), '--standard-json'] compiler_input = json.dumps(json_input) else: assert interface == CompilerInterface.CLI @@ -258,7 +270,7 @@ def prepare_compiler_input( if smt_use == SMTUse.DISABLE: compiler_options += ['--model-checker-engine', 'none'] - command_line = [str(compiler_path)] + compiler_options + command_line += [str(compiler_path)] + compiler_options compiler_input = load_source(source_file_name, smt_use) return (command_line, compiler_input) @@ -289,6 +301,7 @@ def detect_metadata_cli_option_support(compiler_path: Path): def run_compiler( compiler_path: Path, + execution_arch: ExecutionArchitecture, source_file_name: Path, force_no_optimize_yul: bool, interface: CompilerInterface, @@ -298,10 +311,10 @@ def run_compiler( tmp_dir: Path, exit_on_error: bool, ) -> FileReport: - if interface == CompilerInterface.STANDARD_JSON: (command_line, compiler_input) = prepare_compiler_input( compiler_path, + execution_arch, Path(source_file_name.name), force_no_optimize_yul, interface, @@ -325,6 +338,7 @@ def run_compiler( (command_line, compiler_input) = prepare_compiler_input( compiler_path.absolute(), + execution_arch, Path(source_file_name.name), force_no_optimize_yul, interface, @@ -354,6 +368,7 @@ def run_compiler( def generate_report( source_file_names: List[str], compiler_path: Path, + execution_arch: ExecutionArchitecture, interface: CompilerInterface, presets: List[SettingsPreset], smt_use: SMTUse, @@ -373,6 +388,7 @@ def generate_report( try: report = run_compiler( compiler_path, + execution_arch, Path(source_file_name), force_no_optimize_yul, interface, @@ -422,6 +438,13 @@ def commandline_parser() -> ArgumentParser: choices=[c.value for c in CompilerInterface], help="Compiler interface to use.", ) + parser.add_argument( + '--execution-arch', + dest='execution_arch', + default=ExecutionArchitecture.NATIVE.value, + choices=[c.value for c in ExecutionArchitecture], + help="Select the architecture of the universal binary that should be executed. (Only relevant for macOS)", + ) parser.add_argument( '--preset', dest='presets', @@ -470,6 +493,7 @@ def commandline_parser() -> ArgumentParser: generate_report( glob("*.sol"), Path(options.compiler_path), + ExecutionArchitecture(options.execution_arch), CompilerInterface(options.interface), [SettingsPreset(p) for preset_group in presets for p in preset_group], SMTUse(options.smt_use), diff --git a/scripts/install_obsolete_jsoncpp_1_7_4.sh b/scripts/install_obsolete_jsoncpp_1_7_4.sh index 825d1a58a..f9de0245d 100755 --- a/scripts/install_obsolete_jsoncpp_1_7_4.sh +++ b/scripts/install_obsolete_jsoncpp_1_7_4.sh @@ -17,7 +17,7 @@ TEMPDIR=$(mktemp -d) cd "jsoncpp-${jsoncpp_version}" mkdir -p build cd build - cmake -DARCHIVE_INSTALL_DIR=. -G "Unix Makefiles" .. + cmake -DCMAKE_OSX_ARCHITECTURES:STRING="x86_64;arm64" -DARCHIVE_INSTALL_DIR=. -G "Unix Makefiles" .. make make install ) diff --git a/test/scripts/test_bytecodecompare_prepare_report.py b/test/scripts/test_bytecodecompare_prepare_report.py index d73b1f714..28a89ec31 100644 --- a/test/scripts/test_bytecodecompare_prepare_report.py +++ b/test/scripts/test_bytecodecompare_prepare_report.py @@ -9,7 +9,8 @@ # NOTE: This test file file only works with scripts/ added to PYTHONPATH so pylint can't find the imports # pragma pylint: disable=import-error -from bytecodecompare.prepare_report import CompilerInterface, FileReport, ContractReport, SettingsPreset, SMTUse, Statistics +from bytecodecompare.prepare_report import ExecutionArchitecture, CompilerInterface, FileReport, ContractReport +from bytecodecompare.prepare_report import SettingsPreset, SMTUse, Statistics from bytecodecompare.prepare_report import load_source, parse_cli_output, parse_standard_json_output, prepare_compiler_input # pragma pylint: enable=import-error @@ -223,6 +224,7 @@ def test_prepare_compiler_input_should_work_with_standard_json_interface(self): (command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_SMOKE_TEST_SOL_PATH, preset=SettingsPreset.LEGACY_OPTIMIZE, force_no_optimize_yul=False, @@ -237,6 +239,7 @@ def test_prepare_compiler_input_should_work_with_standard_json_interface(self): def test_prepare_compiler_input_should_work_with_cli_interface(self): (command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_SMOKE_TEST_SOL_PATH, preset=SettingsPreset.LEGACY_OPTIMIZE, force_no_optimize_yul=False, @@ -273,6 +276,7 @@ def test_prepare_compiler_input_for_json_preserves_newlines(self): (command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH, preset=SettingsPreset.VIA_IR_OPTIMIZE, force_no_optimize_yul=False, @@ -287,6 +291,7 @@ def test_prepare_compiler_input_for_json_preserves_newlines(self): def test_prepare_compiler_input_for_cli_preserves_newlines(self): (_command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_CONTRACT_WITH_MIXED_NEWLINES_SOL_PATH, preset=SettingsPreset.LEGACY_OPTIMIZE, force_no_optimize_yul=True, @@ -300,6 +305,7 @@ def test_prepare_compiler_input_for_cli_preserves_newlines(self): def test_prepare_compiler_input_for_cli_should_handle_force_no_optimize_yul_flag(self): (command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_SMOKE_TEST_SOL_PATH, preset=SettingsPreset.LEGACY_NO_OPTIMIZE, force_no_optimize_yul=True, @@ -317,6 +323,7 @@ def test_prepare_compiler_input_for_cli_should_handle_force_no_optimize_yul_flag def test_prepare_compiler_input_for_cli_should_not_use_metadata_option_if_not_supported(self): (command_line, compiler_input) = prepare_compiler_input( Path('solc'), + ExecutionArchitecture.NATIVE, SMT_SMOKE_TEST_SOL_PATH, preset=SettingsPreset.VIA_IR_OPTIMIZE, force_no_optimize_yul=False,