From 5c1e8aa3ec98afb29398880cf931839ffb169615 Mon Sep 17 00:00:00 2001 From: JaDoggx86 Date: Sat, 16 Dec 2023 23:27:39 +0000 Subject: [PATCH] build(release,ci,github-ci): add support for macos build --- .github/workflows/release.yml | 2 +- .github/workflows/release_macos.yml | 32 +++++++++++++++ compiler/3rd/reproc_single_file.c | 56 +++++++++++++++++++++++++ compiler/carpntr/build/program_code.c | 2 +- compiler/carpntr/yaksha.toml | 2 +- compiler/hammer.toml | 13 +++--- compiler/scripts/release.ini | 22 +++++++++- compiler/scripts/release.py | 59 +++++++++++++++++++++++---- compiler/src/yk.cpp | 6 +++ 9 files changed, 176 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/release_macos.yml create mode 100644 compiler/3rd/reproc_single_file.c diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 917c8ac4..d9854a2b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,13 +2,13 @@ on: push: tags: - 'v*' + workflow_dispatch: env: PYTHONIOENCODING: utf-8 PYTHONLEGACYWINDOWSSTDIO: utf-8 jobs: release: runs-on: ${{ matrix.os }} - # TODO Add support for running release in macos as well strategy: matrix: os: [windows-2019] diff --git a/.github/workflows/release_macos.yml b/.github/workflows/release_macos.yml new file mode 100644 index 00000000..0bc2555c --- /dev/null +++ b/.github/workflows/release_macos.yml @@ -0,0 +1,32 @@ +on: + push: + tags: + - 'v*' + workflow_dispatch: +env: + PYTHONIOENCODING: utf-8 + PYTHONLEGACYWINDOWSSTDIO: utf-8 +jobs: + release: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-11] + steps: + - name: Install 7z + run: brew install p7zip + - name: cmake + run: brew install cmake + - name: Checkout git repo + uses: actions/checkout@v2 + with: + path: main + submodules: 'true' + - name: Build zips & 7zs + run: python main/compiler/scripts/release.py + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + main/compiler/comp_output_test/yaksha_v*.zip + main/compiler/comp_output_test/yaksha_v*.7z \ No newline at end of file diff --git a/compiler/3rd/reproc_single_file.c b/compiler/3rd/reproc_single_file.c new file mode 100644 index 00000000..5cc71f39 --- /dev/null +++ b/compiler/3rd/reproc_single_file.c @@ -0,0 +1,56 @@ +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__) +#define YK_R_OS_WIN +#elif __APPLE__ +#define YK_R_OS_MAC +#elif __linux__ +#define YK_R_OS_LINUX +#else +#error "Not supported" +#endif + +#ifdef YK_R_OS_WIN +#include "reproc/reproc/src/clock.windows.c" +#else +#include "reproc/reproc/src/clock.posix.c" +#endif + +#include "reproc/reproc/src/drain.c" + + +#ifdef YK_R_OS_WIN +#include "reproc/reproc/src/error.windows.c" +#include "reproc/reproc/src/handle.windows.c" +#include "reproc/reproc/src/init.windows.c" +#else +#include "reproc/reproc/src/error.posix.c" +#include "reproc/reproc/src/handle.posix.c" +#include "reproc/reproc/src/init.posix.c" +#endif + + + +#include "reproc/reproc/src/options.c" + +#ifdef YK_R_OS_WIN +#include "reproc/reproc/src/pipe.windows.c" +#include "reproc/reproc/src/process.windows.c" +#include "reproc/reproc/src/redirect.windows.c" +#else +#include "reproc/reproc/src/pipe.posix.c" +#include "reproc/reproc/src/process.posix.c" +#include "reproc/reproc/src/redirect.posix.c" +#endif + + + +#include "reproc/reproc/src/redirect.c" +#include "reproc/reproc/src/reproc.c" +#include "reproc/reproc/src/run.c" +#include "reproc/reproc/src/strv.c" + + +#ifdef YK_R_OS_WIN +#include "reproc/reproc/src/utf.windows.c" +#else +#include "reproc/reproc/src/utf.posix.c" +#endif diff --git a/compiler/carpntr/build/program_code.c b/compiler/carpntr/build/program_code.c index 433aab27..c9e6ca04 100644 --- a/compiler/carpntr/build/program_code.c +++ b/compiler/carpntr/build/program_code.c @@ -3630,7 +3630,7 @@ yk__sds yy__os_which(yk__sds yy__os_binary) } yk__sds yy__libs_version() { - return yk__sdsnewlen("0.0.4", 5); + return yk__sdsnewlen("0.0.5", 5); } void yy__print_banner() { diff --git a/compiler/carpntr/yaksha.toml b/compiler/carpntr/yaksha.toml index b7acc478..5211f345 100644 --- a/compiler/carpntr/yaksha.toml +++ b/compiler/carpntr/yaksha.toml @@ -22,4 +22,4 @@ libc="try_musl" # Automatically use zig cc & zig c++ to build project # We will start with just zig cc, zig c++ support so this is ignored for now compiler="zig" -targets=["x86_64-linux-musl", "x86_64-windows-gnu"] +targets=["x86_64-linux-musl", "x86_64-windows-gnu", "aarch64-macos-gnu", "x86_64-macos-gnu"] diff --git a/compiler/hammer.toml b/compiler/hammer.toml index 3f36ae75..476c497a 100644 --- a/compiler/hammer.toml +++ b/compiler/hammer.toml @@ -13,22 +13,25 @@ # Note that all the paths are relative in current directory # [compilation] -targets=["x86_64-linux-musl", "x86_64-windows-gnu"] +targets=["x86_64-linux-musl", "x86_64-windows-gnu", "aarch64-macos-gnu", "x86_64-macos-gnu"] args_c_or_cpp=["-O3", "-fPIC"] args_cpp=["-std=c++17"] args_c=["-std=c99"] -include_paths=["src", "3rd/utfcpp/source", "runtime"] +include_paths=["src", "3rd/utfcpp/source", "runtime", + "3rd/reproc/reproc/include", "3rd/reproc/reproc++/include", + "3rd/reproc/reproc/src", "3rd/reproc/reproc++/src", "3rd"] # .c or .cpp files that get compiled to .o files # so this is all except the .c/.cpp file with main() sources=["src/ast/ast.cpp", "src/ast/ast_printer.cpp", "src/ast/ast_vis.cpp", "src/ast/codefiles.cpp", "src/ast/environment.cpp", "src/ast/environment_stack.cpp", "src/ast/parser.cpp", "src/builtins/builtins.cpp", "src/compiler/compiler.cpp", "src/compiler/compiler_utils.cpp", "src/compiler/def_class_visitor.cpp", "src/compiler/delete_stack.cpp", "src/compiler/delete_stack_stack.cpp", "src/compiler/desugaring_compiler.cpp", "src/compiler/entry_struct_func_compiler.cpp", "src/compiler/multifile_compiler.cpp", "src/compiler/return_checker.cpp", "src/compiler/type_checker.cpp", "src/compiler/usage_analyser.cpp", "src/file_formats/ic_tokens_file.cpp", "src/file_formats/tokens_file.cpp", "src/ic2c/ic2c.cpp", "src/ic2c/ic_ast.cpp", "src/ic2c/ic_compiler.cpp", "src/ic2c/ic_level2_parser.cpp", "src/ic2c/ic_level2_tokenizer.cpp", "src/ic2c/ic_line_splicer.cpp", "src/ic2c/ic_optimizer.cpp", "src/ic2c/ic_parser.cpp", "src/ic2c/ic_peek_ahead_iter.cpp", "src/ic2c/ic_preprocessor.cpp", "src/ic2c/ic_tokenizer.cpp", "src/ic2c/ic_trigraph_translater.cpp", "src/tokenizer/block_analyzer.cpp", "src/tokenizer/string_utils.cpp", "src/tokenizer/tokenizer.cpp", "src/utilities/annotation.cpp", "src/utilities/annotations.cpp", "src/utilities/colours.cpp", "src/utilities/cpp_util.cpp", "src/utilities/defer_stack.cpp", "src/utilities/defer_stack_stack.cpp", "src/utilities/error_printer.cpp", "src/utilities/human_id.cpp", "src/utilities/ykdatatype.cpp", "src/utilities/ykdt_pool.cpp", "src/utilities/ykobject.cpp", "src/yaksha_lisp/yaksha_lisp.cpp", "src/yaksha_lisp/yaksha_lisp_builtins.cpp"] # update_makefile.py HAMMER_CPP temp_out_dir="bin/hammer" binaries=["yaksha"] -disable_parallel=false +disable_parallel=true [yaksha] main_file="src/yk.cpp" # Additional sources -sources=["src/carpntr_wrapper.cpp", "runtime/whereami.c"] +sources=["src/carpntr_wrapper.cpp", "runtime/whereami.c", + "3rd/reproc_single_file.c", "3rd/reproc/reproc++/src/reproc.cpp"] name="yaksha" out_dir="bin/hammer.bin" -args_link=["-static", "-O3"] +args_link=["-static", "-O2"] diff --git a/compiler/scripts/release.ini b/compiler/scripts/release.ini index 6129a4be..a64a7626 100644 --- a/compiler/scripts/release.ini +++ b/compiler/scripts/release.ini @@ -1,5 +1,5 @@ [main] -releases=["windows-x86_64", "linux-x86_64"] +releases=["windows-x86_64", "linux-x86_64", "macos-aarch64", "macos-x86_64"] temp=comp_output_test version=0.0.5 @@ -24,6 +24,26 @@ bin__yaksha=["bin/hammer.bin/yaksha-x86_64-linux-musl"] bin__carpntr=["carpntr/build/carpntr-x86_64-linux-musl"] bin__hammer=["hammer/build/hammer-x86_64-linux-musl"] +[macos-aarch64] +zig=https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz +zig_sha256=8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287 +zig_subfolder=zig-macos-aarch64-0.9.1 +exe_suffix= +binaries=["yaksha", "carpntr", "hammer"] +bin__yaksha=["bin/hammer.bin/yaksha-aarch64-macos-gnu"] +bin__carpntr=["carpntr/build/carpntr-aarch64-macos-gnu"] +bin__hammer=["hammer/build/hammer-aarch64-macos-gnu"] + +[macos-x86_64] +zig=https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz +zig_sha256=2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c +zig_subfolder=zig-macos-x86_64-0.9.1 +exe_suffix= +binaries=["yaksha", "carpntr", "hammer"] +bin__yaksha=["bin/hammer.bin/yaksha-x86_64-macos-gnu"] +bin__carpntr=["carpntr/build/carpntr-x86_64-macos-gnu"] +bin__hammer=["hammer/build/hammer-x86_64-macos-gnu"] + [common] folders=["runtime", "libs"] files=["README.md", "LICENSE"] diff --git a/compiler/scripts/release.py b/compiler/scripts/release.py index 7629b601..c784b222 100644 --- a/compiler/scripts/release.py +++ b/compiler/scripts/release.py @@ -39,6 +39,7 @@ import ast import configparser import os +import platform import shutil import subprocess import sys @@ -51,10 +52,19 @@ MAX_EXECUTION_TIME_SEC = 60 * 60 PATHS = [] +MAC_OS = sys.platform.startswith('darw') # noqa WINDOWS_OS = sys.platform.startswith('win') -CURRENT_PLATFORM_KEY = "windows-x86_64" if WINDOWS_OS else "linux-x86_64" +ARM_CPU = platform.processor() == "arm" +ARCH = "aarch64" if ARM_CPU else "x86_64" +CURRENT_PLATFORM_KEY = f"linux-{ARCH}" +if WINDOWS_OS: + CURRENT_PLATFORM_KEY = f"windows-{ARCH}" +if MAC_OS: + CURRENT_PLATFORM_KEY = f"macos-{ARCH}" BUNDLED_ZIG_VERSION = "0.9.1" COMPILER_BINARIES = ["yaksha.exe", "yakshac.exe"] if WINDOWS_OS else ["yaksha", "yakshac"] +MUST_HAVE_BIN = set(["yaksha", "carpntr", "zig"]) # noqa +SCRIPT_STATUS = 0 @contextmanager @@ -181,14 +191,14 @@ def download(url: str, sha256_hash: str) -> (bool, str): def execute(args: list): - fuzz_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", - universal_newlines=True, env=dict(os.environ)) + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", + universal_newlines=True, env=dict(os.environ)) try: - so, se = fuzz_process.communicate(timeout=MAX_EXECUTION_TIME_SEC) - return_value = fuzz_process.returncode + so, se = proc.communicate(timeout=MAX_EXECUTION_TIME_SEC) + return_value = proc.returncode except subprocess.TimeoutExpired: - fuzz_process.kill() - fuzz_process.communicate() + proc.kill() + proc.communicate() print("Timed out - ", Colors.fail(repr(args))) return -1 if return_value != 0: @@ -233,6 +243,7 @@ def package(arch: str, directory: str): def copy_binaries(section: Section, target_location): + global SCRIPT_STATUS binaries = section.binaries suffix = section.exe_suffix for bin_name in binaries: @@ -245,6 +256,10 @@ def copy_binaries(section: Section, target_location): break else: print(Colors.red(bin_name), "❌") + # WHY: + # If we do not have yaksha or carpntr binaries, it is a useless build + if bin_name in MUST_HAVE_BIN: + SCRIPT_STATUS = 1 def create_package(directory, name, ext): @@ -285,7 +300,8 @@ def build_release(name: str): shutil.rmtree(directory) shutil.rmtree(temp) # final message - print(Colors.green("all done."), "🎉") + if SCRIPT_STATUS == 0: + print(Colors.green("all done."), "🎉") def build_releases(): @@ -337,6 +353,19 @@ def compile_carpntr(): with navigate(carpntr_path): with updated_path(): execute([sys.executable, "bootstrap_me.py"]) + # WHY? + # carpntr fails to build as native on Mac (M2 in my case) + # so this fixes it in case that has happened + if MAC_OS and not os.path.exists(os.path.join(carpntr_build_path, "bootstrapped-carpntr")): + shutil.copyfile(os.path.join(carpntr_build_path, f"bootstrapped-carpntr-{ARCH}-macos-gnu"), + os.path.join(carpntr_build_path, "bootstrapped-carpntr")) + execute(["chmod", "+x", os.path.join(carpntr_build_path, "bootstrapped-carpntr")]) + with navigate(carpntr_path): + with updated_path(): + execute(["build/bootstrapped-carpntr"]) + shutil.copyfile(os.path.join(carpntr_build_path, f"carpntr-{ARCH}-macos-gnu"), + os.path.join(carpntr_build_path, "carpntr")) + execute(["chmod", "+x", os.path.join(carpntr_build_path, "carpntr")]) global PATHS PATHS.append(carpntr_build_path) @@ -348,6 +377,11 @@ def compile_hammer(): with navigate(hammer_path): with updated_path(): execute(["carpntr"]) + # Workaround for native build issue on Mac M2 CPU + if MAC_OS and not os.path.exists(os.path.join(hammer_build_path, "hammer")): + shutil.copyfile(os.path.join(hammer_build_path, f"hammer-{ARCH}-macos-gnu"), + os.path.join(hammer_build_path, "hammer")) + execute(["chmod", "+x", os.path.join(hammer_build_path, "hammer")]) global PATHS PATHS.append(hammer_build_path) @@ -376,10 +410,17 @@ def ensure_temp(): print(Colors.fail("Failed to create temp_path:"), temp_path) -if __name__ == "__main__": +def main(): # Set work directory to be that of project root. os.chdir(ROOT) ensure_temp() compile_release() print("-" * 40) build_releases() + if SCRIPT_STATUS != 0: + print(Colors.red("Release failed 💀")) + sys.exit(SCRIPT_STATUS) + + +if __name__ == "__main__": + main() diff --git a/compiler/src/yk.cpp b/compiler/src/yk.cpp index db243831..6a8e71c3 100644 --- a/compiler/src/yk.cpp +++ b/compiler/src/yk.cpp @@ -60,6 +60,12 @@ #define main reloader_main #if !defined(_MSC_VER) && (defined(_WIN32) || defined(_WIN64)) #include "reloader_alt.cpp" +#elif defined(__APPLE__) +int reloader_main(int argc, char** argv) { + std::cerr << colours::red( + "yaksha reload is not supported yet in macos\n"); + return 1; +} #else #include "reloader.cpp" #endif