build

#!/usr/bin/env bash
set -euo pipefail

usage() {
  echo "usage: ./build compiler" >&2
  echo "       ./build toolchain ARCHIVE" >&2
  echo "       ./build lazarus" >&2
  echo "       ./build project.dpr debug|release [--diagnostic-mm]" >&2
  exit 2
}

ROOT=$(cd "$(dirname "$0")" && pwd)
STATE="$ROOT/.moonbot"
TOOLCHAIN="$STATE/toolchain"
IDE_TOOLCHAIN="$TOOLCHAIN/ide"
MM_SOURCE="$ROOT/runtime/mm/mormot.core.fpcx64mm.pas"
LAZARUS_REPOSITORY=https://gitlab.com/freepascal.org/lazarus/lazarus.git
LAZARUS_COMMIT=ce01e71c34866d83f69ea7cd855ae7eabea49f38
new_toolchain=
old_toolchain=
ide_profile=

source "$ROOT/scripts/publish_toolchain.sh"

cleanup_compiler_build() {
  [[ -z "$new_toolchain" ]] || rm -rf "$new_toolchain"
  [[ -z "$ide_profile" ]] || rm -rf "$ide_profile"
  if [[ -n "$old_toolchain" && -e "$TOOLCHAIN" ]]; then
    rm -rf "$old_toolchain"
  fi
}

[[ "$(uname -s)" == Linux && "$(uname -m)" == x86_64 ]] || {
  echo "this driver runs only on Linux x86-64; use build.ps1 on Win64" >&2
  exit 1
}

build_compiler() {
  local bootstrap target_compiler target_fpcmake version_dir
  local compiler_options='-O2 -dMOONCOMPILER_PRODUCT_RUNTIME -dMOONCOMPILER_VANILLA_RUNTIME'
  bootstrap=${MOONBOT_BOOTSTRAP_FPC:-$(command -v fpc || true)}
  [[ -n "$bootstrap" && -x "$bootstrap" ]] || {
    echo "FPC 3.2.2 is required once; set MOONBOT_BOOTSTRAP_FPC" >&2
    exit 1
  }
  [[ "$($bootstrap -iV)" == 3.2.2 ]] || {
    echo "bootstrap compiler must be FPC 3.2.2" >&2
    exit 1
  }

  mkdir -p "$STATE"
  new_toolchain="$STATE/toolchain.new.$$"
  old_toolchain="$STATE/toolchain.old.$$"
  ide_profile="$STATE/ide-profile.new.$$"
  trap cleanup_compiler_build EXIT
  rm -rf "$new_toolchain" "$old_toolchain" "$ide_profile"

  make -C "$ROOT" clean
  make -C "$ROOT" -j1 all FPC="$bootstrap" OPT="$compiler_options"
  make -C "$ROOT" install FPC="$bootstrap" OPT="$compiler_options" \
    INSTALL_PREFIX="$new_toolchain"

  version_dir=$(find "$new_toolchain/lib/fpc" -mindepth 1 -maxdepth 1 \
    -type d -name '[0-9]*' -printf '%f\n')
  [[ -n "$version_dir" && "$version_dir" != *$'\n'* ]] || {
    echo "could not identify the installed compiler version" >&2
    exit 1
  }
  target_compiler="$new_toolchain/lib/fpc/$version_dir/ppcx64"
  [[ -x "$target_compiler" ]] || {
    echo "the installed Linux compiler is incomplete" >&2
    exit 1
  }
  target_fpcmake="$ROOT/utils/fpcm/bin/x86_64-linux/fpcmake"
  [[ -x "$target_fpcmake" ]] || {
    echo "the built Linux fpcmake utility is missing" >&2
    exit 1
  }
  [[ -x "$new_toolchain/bin/fpcres" ]] || {
    echo 'the Linux resource compiler was not installed' >&2
    exit 1
  }

  # Preserve the ordinary FPC ABI before the product Unicode RTL replaces
  # the installed units. Lazarus, LCL and compiler tools use this profile.
  cp -a "$new_toolchain" "$ide_profile"

  # Compiler/IDE tools keep their host representation.  The target RTL and
  # application-facing packages use the modern Delphi Unicode ABI.
  make -C "$ROOT/rtl" clean FPC="$target_compiler"
  make -C "$ROOT/rtl" -j1 all FPC="$target_compiler" \
    OPT='-O2 -dMOONCOMPILER_VANILLA_RUNTIME -dUNICODERTL -dENABLE_DELPHI_RTTI -dMOONCOMPILER_DELPHI_CALLBACK_TYPES'
  make -C "$ROOT/rtl" install FPC="$target_compiler" \
    FPCMAKE="$target_fpcmake" OPT='-O2 -dMOONCOMPILER_VANILLA_RUNTIME -dUNICODERTL -dENABLE_DELPHI_RTTI -dMOONCOMPILER_DELPHI_CALLBACK_TYPES' \
    INSTALL_PREFIX="$new_toolchain"
  make -C "$ROOT/packages" clean FPC="$target_compiler" \
    FPMAKEOPT=--NoIDE=1
  make -C "$ROOT/packages" -j1 all FPC="$target_compiler" \
    OPT='-O2 -dMOONCOMPILER_VANILLA_RUNTIME -dUNICODERTL -dENABLE_DELPHI_RTTI -dMOONCOMPILER_DELPHI_CALLBACK_TYPES' FPMAKEOPT=--NoIDE=1
  make -C "$ROOT/packages" install FPC="$target_compiler" \
    FPCMAKE="$target_fpcmake" OPT='-O2 -dMOONCOMPILER_VANILLA_RUNTIME -dUNICODERTL -dENABLE_DELPHI_RTTI -dMOONCOMPILER_DELPHI_CALLBACK_TYPES' FPMAKEOPT=--NoIDE=1 \
    INSTALL_PREFIX="$new_toolchain"

  ln -s "../lib/fpc/$version_dir/ppcx64" "$new_toolchain/bin/ppcx64"
  mkdir -p "$new_toolchain/etc"
  "$new_toolchain/bin/fpcmkcfg" \
    -d "basepath=$TOOLCHAIN/lib/fpc/$version_dir" \
    -o "$new_toolchain/etc/fpc.cfg"
  printf '%s\n' \
    '# MoonCompiler project ABI: Delphi String and Char are Unicode.' \
    '-dMOONCOMPILER_UNICODE_DEFAULT' \
    '# Product programs receive the bundled runtime prefix automatically.' \
    '-dMOONBOT_MM_PROFILE_REQUIRED' \
    '-dFPCMM_BOOSTER' \
    '-dFPCMM_MOONSHARD' \
    "--pinned-unit=mormot.core.fpcx64mm=$MM_SOURCE" \
    >> "$new_toolchain/etc/fpc.cfg"

  ln -sf "../lib/fpc/$version_dir/ppcx64" "$ide_profile/bin/ppcx64"
  mkdir -p "$ide_profile/etc"
  "$new_toolchain/bin/fpcmkcfg" \
    -d "basepath=$IDE_TOOLCHAIN/lib/fpc/$version_dir" \
    -o "$ide_profile/etc/fpc.cfg"
  printf '%s\n' \
    '# MoonCompiler IDE ABI: ordinary FPC String and Char representation.' \
    '# Do not inject the product memory manager or runtime prefix.' \
    '-dMOONCOMPILER_VANILLA_RUNTIME' \
    >> "$ide_profile/etc/fpc.cfg"

  mkdir -p "$new_toolchain/share/doc/mooncompiler"
  cp "$ROOT/compiler/COPYING.txt" \
    "$new_toolchain/share/doc/mooncompiler/COMPILER-GPL-2.0.txt"
  cp "$ROOT/rtl/COPYING.txt" \
    "$new_toolchain/share/doc/mooncompiler/RTL-LGPL-2.1.txt"
  cp "$ROOT/rtl/COPYING.FPC" \
    "$new_toolchain/share/doc/mooncompiler/RTL-EXCEPTION.txt"
  cp "$ROOT/runtime/mm/LICENSE.md" \
    "$new_toolchain/share/doc/mooncompiler/MM-LICENSE.md"
  cp "$ROOT/doc/LICENSING.md" \
    "$new_toolchain/share/doc/mooncompiler/LICENSING.md"

  mv "$ide_profile" "$new_toolchain/ide"
  ide_profile=

  publish_toolchain "$new_toolchain" "$TOOLCHAIN" "$old_toolchain"
  trap - EXIT
  echo "MoonCompiler installed in $TOOLCHAIN"
}

install_toolchain() {
  local archive member link target version_dir target_cpu target_os
  local product_version ide_version
  archive=$(realpath -- "$1")
  [[ -f "$archive" ]] || {
    echo "toolchain archive does not exist: $archive" >&2
    exit 1
  }
  case "$archive" in
    *.tar.gz|*.tgz) ;;
    *)
      echo 'Linux toolchain archive must be a .tar.gz or .tgz file' >&2
      exit 1
      ;;
  esac

  # Reject archive entries that could escape the transactional staging tree.
  while IFS= read -r member; do
    member=${member#./}
    [[ -z "$member" ]] && continue
    [[ "$member" == /* ||
       "$member" == .. || "$member" == ../* || "$member" == */../* ||
       "$member" == */.. ]] && {
      echo "unsafe path in toolchain archive: $member" >&2
      exit 1
    }
  done < <(tar -tzf "$archive")

  mkdir -p "$STATE"
  new_toolchain="$STATE/toolchain.new.$$"
  old_toolchain="$STATE/toolchain.old.$$"
  trap cleanup_compiler_build EXIT
  rm -rf "$new_toolchain" "$old_toolchain"
  mkdir "$new_toolchain"
  tar -xzf "$archive" --no-same-owner --no-same-permissions \
    -C "$new_toolchain"

  [[ -x "$new_toolchain/bin/fpc" &&
     -x "$new_toolchain/bin/fpcmkcfg" &&
     -x "$new_toolchain/bin/fpcres" &&
     -x "$new_toolchain/ide/bin/fpc" &&
     -x "$new_toolchain/ide/bin/fpcres" &&
     -d "$new_toolchain/lib/fpc" &&
     -d "$new_toolchain/ide/lib/fpc" &&
     -f "$new_toolchain/share/doc/mooncompiler/COMPILER-GPL-2.0.txt" &&
     -f "$new_toolchain/share/doc/mooncompiler/RTL-LGPL-2.1.txt" &&
     -f "$new_toolchain/share/doc/mooncompiler/RTL-EXCEPTION.txt" &&
     -f "$new_toolchain/share/doc/mooncompiler/MM-LICENSE.md" &&
     -f "$new_toolchain/share/doc/mooncompiler/LICENSING.md" ]] || {
    echo 'the archive is not a complete Linux x86-64 MoonCompiler toolchain' >&2
    exit 1
  }
  target_cpu=$($new_toolchain/bin/fpc -iTP)
  target_os=$($new_toolchain/bin/fpc -iTO)
  product_version=$($new_toolchain/bin/fpc -iV)
  ide_version=$($new_toolchain/ide/bin/fpc -iV)
  [[ "$target_cpu" == x86_64 && "$target_os" == linux &&
     "$product_version" == "$ide_version" ]] || {
    echo 'the archive compiler target or IDE profile version is invalid' >&2
    exit 1
  }
  while IFS= read -r -d '' link; do
    target=$(readlink -m -- "$(dirname "$link")/$(readlink "$link")")
    [[ "$target" == "$new_toolchain"/* ]] || {
      echo "unsafe symbolic link in toolchain archive: $link" >&2
      exit 1
    }
  done < <(find "$new_toolchain" -type l -print0)

  version_dir=$(find "$new_toolchain/lib/fpc" -mindepth 1 -maxdepth 1 \
    -type d -name '[0-9]*' -printf '%f\n')
  [[ -n "$version_dir" && "$version_dir" != *$'\n'* &&
     -d "$new_toolchain/ide/lib/fpc/$version_dir" ]] || {
    echo 'could not identify one matching compiler version in the archive' >&2
    exit 1
  }

  mkdir -p "$new_toolchain/etc" "$new_toolchain/ide/etc"
  "$new_toolchain/bin/fpcmkcfg" \
    -d "basepath=$TOOLCHAIN/lib/fpc/$version_dir" \
    -o "$new_toolchain/etc/fpc.cfg"
  printf '%s\n' \
    '# MoonCompiler project ABI: Delphi String and Char are Unicode.' \
    '-dMOONCOMPILER_UNICODE_DEFAULT' \
    '# Product programs receive the bundled runtime prefix automatically.' \
    '-dMOONBOT_MM_PROFILE_REQUIRED' \
    '-dFPCMM_BOOSTER' \
    '-dFPCMM_MOONSHARD' \
    "--pinned-unit=mormot.core.fpcx64mm=$MM_SOURCE" \
    >> "$new_toolchain/etc/fpc.cfg"
  "$new_toolchain/bin/fpcmkcfg" \
    -d "basepath=$IDE_TOOLCHAIN/lib/fpc/$version_dir" \
    -o "$new_toolchain/ide/etc/fpc.cfg"
  printf '%s\n' \
    '# MoonCompiler IDE ABI: ordinary FPC String and Char representation.' \
    '# Do not inject the product memory manager or runtime prefix.' \
    '-dMOONCOMPILER_VANILLA_RUNTIME' \
    >> "$new_toolchain/ide/etc/fpc.cfg"

  publish_toolchain "$new_toolchain" "$TOOLCHAIN" "$old_toolchain"
  trap - EXIT
  echo "MoonCompiler toolchain installed in $TOOLCHAIN"
}

build_lazarus() {
  local ide_fpc source head pcp lazbuild_options
  ide_fpc="$IDE_TOOLCHAIN/bin/fpc"
  [[ -x "$ide_fpc" ]] || {
    echo 'the IDE profile is missing; run ./build compiler first' >&2
    exit 1
  }
  command -v git >/dev/null || {
    echo 'Git is required to fetch the pinned Lazarus source' >&2
    exit 1
  }
  if ! command -v pkg-config >/dev/null ||
     ! pkg-config --exists gtk+-3.0; then
    echo 'GTK3 development files are required to build Lazarus' >&2
    echo 'Debian/Ubuntu: sudo apt install libgtk-3-dev' >&2
    exit 1
  fi
  source="$STATE/lazarus"
  if [[ ! -d "$source/.git" ]]; then
    [[ ! -e "$source" ]] || {
      echo "the managed Lazarus directory exists but is not a Git checkout: $source" >&2
      exit 1
    }
    git clone --filter=blob:none --no-checkout "$LAZARUS_REPOSITORY" "$source"
    git -C "$source" checkout --detach "$LAZARUS_COMMIT"
  fi
  head=$(git -C "$source" rev-parse HEAD)
  [[ "$head" == "$LAZARUS_COMMIT" ]] || {
    echo "managed Lazarus checkout is not at the supported commit $LAZARUS_COMMIT" >&2
    exit 1
  }

  pcp="$STATE/lazarus-config"
  mkdir -p "$pcp"
  lazbuild_options='--lazarusdir=. --compiler=$(PP) --cpu=$(CPU_TARGET) --os=$(OS_TARGET) --opt="$(OPT)" --pcp="$(MOON_LAZARUS_PCP)"'

  env -u FPCDIR PATH="$IDE_TOOLCHAIN/bin:$PATH" PP="$ide_fpc" \
    PPC_CONFIG_PATH="$IDE_TOOLCHAIN/etc" \
    LAZARUSDIR="$source" \
    make -C "$source" clean FPC="$ide_fpc"
  env -u FPCDIR PATH="$IDE_TOOLCHAIN/bin:$PATH" PP="$ide_fpc" \
    PPC_CONFIG_PATH="$IDE_TOOLCHAIN/etc" \
    LAZARUSDIR="$source" \
    make -C "$source" -j1 bigide FPC="$ide_fpc" OPT=-O3 \
      MOON_LAZARUS_PCP="$pcp" "LAZBUILDOPTS=$lazbuild_options"
  [[ -x "$source/lazarus" ]] || {
    echo 'the Lazarus build finished without the lazarus executable' >&2
    exit 1
  }

  xml_escape() {
    printf '%s' "$1" |
      sed -e 's/&/\&amp;/g' -e 's/"/\&quot;/g' \
        -e 's/</\&lt;/g' -e 's/>/\&gt;/g'
  }
  cat > "$pcp/environmentoptions.xml" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<CONFIG>
  <EnvironmentOptions>
    <LazarusDirectory Value="$(xml_escape "$source")"/>
    <CompilerFilename Value="$(xml_escape "$ide_fpc")"/>
    <FPCSourceDirectory Value="$(xml_escape "$ROOT")"/>
    <MakeFilename Value="$(xml_escape "$(command -v make)")"/>
    <Version Value="112" Lazarus="4.99"/>
  </EnvironmentOptions>
</CONFIG>
EOF
  echo "Lazarus $LAZARUS_COMMIT built in $source"
  echo 'Launch it with ./lazarus'
}

if [[ $# -eq 1 && "$1" == compiler ]]; then
  build_compiler
  exit 0
fi

if [[ $# -eq 2 && "$1" == toolchain ]]; then
  install_toolchain "$2"
  exit 0
fi

if [[ $# -eq 1 && "$1" == lazarus ]]; then
  build_lazarus
  exit 0
fi

[[ $# -ge 2 && $# -le 3 ]] || usage
project=$1
profile=$2
diagnostic_mm=${3:-}
[[ -f "$project" && "$project" == *.dpr ]] || usage
case "$profile" in
  debug|release) ;;
  *) usage ;;
esac
[[ -z "$diagnostic_mm" || "$diagnostic_mm" == --diagnostic-mm ]] || usage
[[ -x "$TOOLCHAIN/bin/fpc" && -f "$TOOLCHAIN/etc/fpc.cfg" ]] || {
  echo "MoonCompiler is not built; run ./build compiler" >&2
  exit 1
}

case "$project" in
  /*) ;;
  *) project="$PWD/$project" ;;
esac
project_name=${project##*/}
project_dir=${project%/*}
project_dir=$(cd -L -- "$project_dir" && pwd -L)
project="$project_dir/$project_name"
project_id=$(printf '%s' "$project" | sha256sum | cut -d' ' -f1)
runtime_mode=normal
diagnostic_options=()
if [[ "$diagnostic_mm" == --diagnostic-mm ]]; then
  runtime_mode=diagnostic-mm
  diagnostic_options=(-dFPCX64MM_DIAGNOSTIC)
fi
unit_dir="$STATE/units/linux/$project_id/$profile/$runtime_mode"
mkdir -p "$unit_dir"

namespace_options=(
  -FNSystem
  -UaSystem.SysUtils=SysUtils
  -UaSystem.Variants=Variants
  -UaSystem.Classes=Classes
  -UaSystem.DateUtils=DateUtils
  -UaSystem.Math=Math
  -UaSystem.Types=Types
  -UaSystem.TypInfo=TypInfo
  -UaSystem.Rtti=Rtti
  -UaSystem.StrUtils=StrUtils
  -UaSystem.Character=Character
  -UaSystem.SyncObjs=SyncObjs
  -UaSystem.Generics.Defaults=Generics.Defaults
  -UaSystem.Generics.Collections=Generics.Collections
  -UaSystem.IniFiles=IniFiles
  -UaSystem.SysConst=SysConst
  -UaSystem.RTLConsts=RTLConsts
)
if [[ "$profile" == debug ]]; then
  profile_options=(-dDEBUG -uRELEASE -O- -gl -gw3 -Ci -Co- -Cr- -Ct- -Sa)
else
  profile_options=(-dRELEASE -uDEBUG -O3 -gl -gw3 -Ci -Co- -Cr- -Ct- -Sa-)
fi

options=(
  -n "@$TOOLCHAIN/etc/fpc.cfg" -Mdelphi
  -Municodestrings -MduplicateLocals -Madvancedrecords -Marrayoperators
  -Munderscoreisseparator -Mfunctionreferences -Manonymousfunctions
  -Minlinevars -Mimplicitgenerics -Mautoderef
  -Px86_64 -Tlinux -Rintel -dPOSIX -B
  -dMOONBOT_MM_PROFILE_REQUIRED -dFPCMM_BOOSTER -dFPCMM_MOONSHARD
  "--pinned-unit=mormot.core.fpcx64mm=$MM_SOURCE"
  "${namespace_options[@]}"
  "-FU$unit_dir/app" "-FE$project_dir"
)
mkdir -p "$unit_dir/app"
source "$ROOT/scripts/project_profile.sh"
configure_project_profile
options+=("${profile_options[@]}")
options+=("${diagnostic_options[@]}")

echo "building $project ($profile, $runtime_mode, Linux x86-64)"
"$TOOLCHAIN/bin/fpc" "${options[@]}" "$project"