build(release,ci,github-ci): add support for macos build

.github/workflows/release.yml

@@ -2,13 +2,13 @@ on:
   push:
     tags:
       - 'v*'
+  workflow_dispatch:
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
 jobs:
   release:
     runs-on: ${{ matrix.os }}
-    # TODO Add support for running release in macos as well
     strategy:
       matrix:
         os: [windows-2019]

.github/workflows/release_macos.yml

@@ -0,0 +1,32 @@
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+env:
+  PYTHONIOENCODING: utf-8
+  PYTHONLEGACYWINDOWSSTDIO: utf-8
+jobs:
+  release:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-11]
+    steps:
+    - name: Install 7z
+      run: brew install p7zip
+    - name: cmake
+      run: brew install cmake
+    - name: Checkout git repo
+      uses: actions/checkout@v2
+      with:
+        path: main
+        submodules: 'true'
+    - name: Build zips & 7zs
+      run: python main/compiler/scripts/release.py
+    - name: Release
+      uses: softprops/action-gh-release@v1
+      with:
+        files: |
+          main/compiler/comp_output_test/yaksha_v*.zip
+          main/compiler/comp_output_test/yaksha_v*.7z

compiler/3rd/reproc_single_file.c

@@ -0,0 +1,56 @@
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
+#define YK_R_OS_WIN
+#elif __APPLE__
+#define YK_R_OS_MAC
+#elif __linux__
+#define YK_R_OS_LINUX
+#else
+#error "Not supported"
+#endif
+
+#ifdef YK_R_OS_WIN
+#include "reproc/reproc/src/clock.windows.c"
+#else
+#include "reproc/reproc/src/clock.posix.c"
+#endif
+
+#include "reproc/reproc/src/drain.c"
+
+
+#ifdef YK_R_OS_WIN
+#include "reproc/reproc/src/error.windows.c"
+#include "reproc/reproc/src/handle.windows.c"
+#include "reproc/reproc/src/init.windows.c"
+#else
+#include "reproc/reproc/src/error.posix.c"
+#include "reproc/reproc/src/handle.posix.c"
+#include "reproc/reproc/src/init.posix.c"
+#endif
+
+
+
+#include "reproc/reproc/src/options.c"
+
+#ifdef YK_R_OS_WIN
+#include "reproc/reproc/src/pipe.windows.c"
+#include "reproc/reproc/src/process.windows.c"
+#include "reproc/reproc/src/redirect.windows.c"
+#else
+#include "reproc/reproc/src/pipe.posix.c"
+#include "reproc/reproc/src/process.posix.c"
+#include "reproc/reproc/src/redirect.posix.c"
+#endif
+
+
+
+#include "reproc/reproc/src/redirect.c"
+#include "reproc/reproc/src/reproc.c"
+#include "reproc/reproc/src/run.c"
+#include "reproc/reproc/src/strv.c"
+
+
+#ifdef YK_R_OS_WIN
+#include "reproc/reproc/src/utf.windows.c"
+#else
+#include "reproc/reproc/src/utf.posix.c"
+#endif

compiler/carpntr/build/program_code.c

@@ -3630,7 +3630,7 @@ yk__sds yy__os_which(yk__sds yy__os_binary)
 }
 yk__sds yy__libs_version() 
 {
-    return yk__sdsnewlen("0.0.4", 5);
+    return yk__sdsnewlen("0.0.5", 5);
 }
 void yy__print_banner() 
 {

compiler/carpntr/yaksha.toml

@@ -22,4 +22,4 @@ libc="try_musl"
 # Automatically use zig cc & zig c++ to build project
 # We will start with just zig cc, zig c++ support so this is ignored for now
 compiler="zig"
-targets=["x86_64-linux-musl", "x86_64-windows-gnu"]
+targets=["x86_64-linux-musl", "x86_64-windows-gnu", "aarch64-macos-gnu", "x86_64-macos-gnu"]

compiler/hammer.toml

@@ -13,22 +13,25 @@
 # Note that all the paths are relative in current directory
 #
 [compilation]
-targets=["x86_64-linux-musl", "x86_64-windows-gnu"]
+targets=["x86_64-linux-musl", "x86_64-windows-gnu", "aarch64-macos-gnu", "x86_64-macos-gnu"]
 args_c_or_cpp=["-O3", "-fPIC"]
 args_cpp=["-std=c++17"]
 args_c=["-std=c99"]
-include_paths=["src", "3rd/utfcpp/source", "runtime"]
+include_paths=["src", "3rd/utfcpp/source", "runtime",
+    "3rd/reproc/reproc/include", "3rd/reproc/reproc++/include",
+    "3rd/reproc/reproc/src", "3rd/reproc/reproc++/src", "3rd"]
 # .c or .cpp files that get compiled to .o files
 # so this is all except the .c/.cpp file with main()
 sources=["src/ast/ast.cpp", "src/ast/ast_printer.cpp", "src/ast/ast_vis.cpp", "src/ast/codefiles.cpp", "src/ast/environment.cpp", "src/ast/environment_stack.cpp", "src/ast/parser.cpp", "src/builtins/builtins.cpp", "src/compiler/compiler.cpp", "src/compiler/compiler_utils.cpp", "src/compiler/def_class_visitor.cpp", "src/compiler/delete_stack.cpp", "src/compiler/delete_stack_stack.cpp", "src/compiler/desugaring_compiler.cpp", "src/compiler/entry_struct_func_compiler.cpp", "src/compiler/multifile_compiler.cpp", "src/compiler/return_checker.cpp", "src/compiler/type_checker.cpp", "src/compiler/usage_analyser.cpp", "src/file_formats/ic_tokens_file.cpp", "src/file_formats/tokens_file.cpp", "src/ic2c/ic2c.cpp", "src/ic2c/ic_ast.cpp", "src/ic2c/ic_compiler.cpp", "src/ic2c/ic_level2_parser.cpp", "src/ic2c/ic_level2_tokenizer.cpp", "src/ic2c/ic_line_splicer.cpp", "src/ic2c/ic_optimizer.cpp", "src/ic2c/ic_parser.cpp", "src/ic2c/ic_peek_ahead_iter.cpp", "src/ic2c/ic_preprocessor.cpp", "src/ic2c/ic_tokenizer.cpp", "src/ic2c/ic_trigraph_translater.cpp", "src/tokenizer/block_analyzer.cpp", "src/tokenizer/string_utils.cpp", "src/tokenizer/tokenizer.cpp", "src/utilities/annotation.cpp", "src/utilities/annotations.cpp", "src/utilities/colours.cpp", "src/utilities/cpp_util.cpp", "src/utilities/defer_stack.cpp", "src/utilities/defer_stack_stack.cpp", "src/utilities/error_printer.cpp", "src/utilities/human_id.cpp", "src/utilities/ykdatatype.cpp", "src/utilities/ykdt_pool.cpp", "src/utilities/ykobject.cpp", "src/yaksha_lisp/yaksha_lisp.cpp", "src/yaksha_lisp/yaksha_lisp_builtins.cpp"] # update_makefile.py HAMMER_CPP
 temp_out_dir="bin/hammer"
 binaries=["yaksha"]
-disable_parallel=false
+disable_parallel=true
 
 [yaksha]
 main_file="src/yk.cpp"
 # Additional sources
-sources=["src/carpntr_wrapper.cpp", "runtime/whereami.c"]
+sources=["src/carpntr_wrapper.cpp", "runtime/whereami.c",
+    "3rd/reproc_single_file.c", "3rd/reproc/reproc++/src/reproc.cpp"]
 name="yaksha"
 out_dir="bin/hammer.bin"
-args_link=["-static", "-O3"]
+args_link=["-static", "-O2"]

compiler/scripts/release.ini

@@ -1,5 +1,5 @@
 [main]
-releases=["windows-x86_64", "linux-x86_64"]
+releases=["windows-x86_64", "linux-x86_64", "macos-aarch64", "macos-x86_64"]
 temp=comp_output_test
 version=0.0.5
 
@@ -24,6 +24,26 @@ bin__yaksha=["bin/hammer.bin/yaksha-x86_64-linux-musl"]
 bin__carpntr=["carpntr/build/carpntr-x86_64-linux-musl"]
 bin__hammer=["hammer/build/hammer-x86_64-linux-musl"]
 
+[macos-aarch64]
+zig=https://ziglang.org/download/0.9.1/zig-macos-aarch64-0.9.1.tar.xz
+zig_sha256=8c473082b4f0f819f1da05de2dbd0c1e891dff7d85d2c12b6ee876887d438287
+zig_subfolder=zig-macos-aarch64-0.9.1
+exe_suffix=
+binaries=["yaksha", "carpntr", "hammer"]
+bin__yaksha=["bin/hammer.bin/yaksha-aarch64-macos-gnu"]
+bin__carpntr=["carpntr/build/carpntr-aarch64-macos-gnu"]
+bin__hammer=["hammer/build/hammer-aarch64-macos-gnu"]
+
+[macos-x86_64]
+zig=https://ziglang.org/download/0.9.1/zig-macos-x86_64-0.9.1.tar.xz
+zig_sha256=2d94984972d67292b55c1eb1c00de46580e9916575d083003546e9a01166754c
+zig_subfolder=zig-macos-x86_64-0.9.1
+exe_suffix=
+binaries=["yaksha", "carpntr", "hammer"]
+bin__yaksha=["bin/hammer.bin/yaksha-x86_64-macos-gnu"]
+bin__carpntr=["carpntr/build/carpntr-x86_64-macos-gnu"]
+bin__hammer=["hammer/build/hammer-x86_64-macos-gnu"]
+
 [common]
 folders=["runtime", "libs"]
 files=["README.md", "LICENSE"]

compiler/scripts/release.py

@@ -39,6 +39,7 @@
 import ast
 import configparser
 import os
+import platform
 import shutil
 import subprocess
 import sys
@@ -51,10 +52,19 @@
 MAX_EXECUTION_TIME_SEC = 60 * 60
 PATHS = []
 
+MAC_OS = sys.platform.startswith('darw')  # noqa
 WINDOWS_OS = sys.platform.startswith('win')
-CURRENT_PLATFORM_KEY = "windows-x86_64" if WINDOWS_OS else "linux-x86_64"
+ARM_CPU = platform.processor() == "arm"
+ARCH = "aarch64" if ARM_CPU else "x86_64"
+CURRENT_PLATFORM_KEY = f"linux-{ARCH}"
+if WINDOWS_OS:
+    CURRENT_PLATFORM_KEY = f"windows-{ARCH}"
+if MAC_OS:
+    CURRENT_PLATFORM_KEY = f"macos-{ARCH}"
 BUNDLED_ZIG_VERSION = "0.9.1"
 COMPILER_BINARIES = ["yaksha.exe", "yakshac.exe"] if WINDOWS_OS else ["yaksha", "yakshac"]
+MUST_HAVE_BIN = set(["yaksha", "carpntr", "zig"])  # noqa
+SCRIPT_STATUS = 0
 
 
 @contextmanager
@@ -181,14 +191,14 @@ def download(url: str, sha256_hash: str) -> (bool, str):
 
 
 def execute(args: list):
-    fuzz_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8",
-                                    universal_newlines=True, env=dict(os.environ))
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8",
+                            universal_newlines=True, env=dict(os.environ))
     try:
-        so, se = fuzz_process.communicate(timeout=MAX_EXECUTION_TIME_SEC)
-        return_value = fuzz_process.returncode
+        so, se = proc.communicate(timeout=MAX_EXECUTION_TIME_SEC)
+        return_value = proc.returncode
     except subprocess.TimeoutExpired:
-        fuzz_process.kill()
-        fuzz_process.communicate()
+        proc.kill()
+        proc.communicate()
         print("Timed out - ", Colors.fail(repr(args)))
         return -1
     if return_value != 0:
@@ -233,6 +243,7 @@ def package(arch: str, directory: str):
 
 
 def copy_binaries(section: Section, target_location):
+    global SCRIPT_STATUS
     binaries = section.binaries
     suffix = section.exe_suffix
     for bin_name in binaries:
@@ -245,6 +256,10 @@ def copy_binaries(section: Section, target_location):
                 break
         else:
             print(Colors.red(bin_name), "❌")
+            # WHY:
+            # If we do not have yaksha or carpntr binaries, it is a useless build
+            if bin_name in MUST_HAVE_BIN:
+                SCRIPT_STATUS = 1
 
 
 def create_package(directory, name, ext):
@@ -285,7 +300,8 @@ def build_release(name: str):
     shutil.rmtree(directory)
     shutil.rmtree(temp)
     # final message
-    print(Colors.green("all done."), "🎉")
+    if SCRIPT_STATUS == 0:
+        print(Colors.green("all done."), "🎉")
 
 
 def build_releases():
@@ -337,6 +353,19 @@ def compile_carpntr():
     with navigate(carpntr_path):
         with updated_path():
             execute([sys.executable, "bootstrap_me.py"])
+    # WHY?
+    # carpntr fails to build as native on Mac (M2 in my case)
+    # so this fixes it in case that has happened
+    if MAC_OS and not os.path.exists(os.path.join(carpntr_build_path, "bootstrapped-carpntr")):
+        shutil.copyfile(os.path.join(carpntr_build_path, f"bootstrapped-carpntr-{ARCH}-macos-gnu"),
+                        os.path.join(carpntr_build_path, "bootstrapped-carpntr"))
+        execute(["chmod", "+x", os.path.join(carpntr_build_path, "bootstrapped-carpntr")])
+        with navigate(carpntr_path):
+            with updated_path():
+                execute(["build/bootstrapped-carpntr"])
+        shutil.copyfile(os.path.join(carpntr_build_path, f"carpntr-{ARCH}-macos-gnu"),
+                        os.path.join(carpntr_build_path, "carpntr"))
+        execute(["chmod", "+x", os.path.join(carpntr_build_path, "carpntr")])
     global PATHS
     PATHS.append(carpntr_build_path)
 
@@ -348,6 +377,11 @@ def compile_hammer():
     with navigate(hammer_path):
         with updated_path():
             execute(["carpntr"])
+    # Workaround for native build issue on Mac M2 CPU
+    if MAC_OS and not os.path.exists(os.path.join(hammer_build_path, "hammer")):
+        shutil.copyfile(os.path.join(hammer_build_path, f"hammer-{ARCH}-macos-gnu"),
+                        os.path.join(hammer_build_path, "hammer"))
+        execute(["chmod", "+x", os.path.join(hammer_build_path, "hammer")])
     global PATHS
     PATHS.append(hammer_build_path)
 
@@ -376,10 +410,17 @@ def ensure_temp():
         print(Colors.fail("Failed to create temp_path:"), temp_path)
 
 
-if __name__ == "__main__":
+def main():
     # Set work directory to be that of project root.
     os.chdir(ROOT)
     ensure_temp()
     compile_release()
     print("-" * 40)
     build_releases()
+    if SCRIPT_STATUS != 0:
+        print(Colors.red("Release failed 💀"))
+    sys.exit(SCRIPT_STATUS)
+
+
+if __name__ == "__main__":
+    main()

compiler/src/yk.cpp

@@ -60,6 +60,12 @@
 #define main reloader_main
 #if !defined(_MSC_VER) && (defined(_WIN32) || defined(_WIN64))
 #include "reloader_alt.cpp"
+#elif defined(__APPLE__)
+int reloader_main(int argc, char** argv) {
+    std::cerr << colours::red(
+            "yaksha reload is not supported yet in macos\n");
+    return 1;
+}
 #else
 #include "reloader.cpp"
 #endif