From 84c6f1e7d0140de999823739e2c7bd701125e7ca Mon Sep 17 00:00:00 2001 From: DellaBitta Date: Fri, 22 Jul 2022 08:37:54 -0400 Subject: [PATCH] Add CI Android targets (#105) Adds a CI Workflow to build the testapps against the publicly available packaged SDK. --- .github/workflows/android.yml | 205 ++++- analytics/testapp/build.gradle | 2 +- dynamic_links/testapp/build.gradle | 2 +- .../build_scripts/android/install_prereqs.sh | 75 ++ scripts/build_scripts/build_testapps.json | 183 +++++ scripts/build_scripts/build_testapps.py | 718 ++++++++++++++++++ scripts/build_scripts/config_reader.py | 139 ++++ scripts/build_scripts/python_requirements.txt | 2 + scripts/build_scripts/utils.py | 94 +++ scripts/build_scripts/xcodebuild.py | 119 +++ scripts/restore_secrets.py | 186 +++++ 11 files changed, 1718 insertions(+), 7 deletions(-) create mode 100755 scripts/build_scripts/android/install_prereqs.sh create mode 100644 scripts/build_scripts/build_testapps.json create mode 100644 scripts/build_scripts/build_testapps.py create mode 100644 scripts/build_scripts/config_reader.py create mode 100644 scripts/build_scripts/python_requirements.txt create mode 100644 scripts/build_scripts/utils.py create mode 100644 scripts/build_scripts/xcodebuild.py create mode 100644 scripts/restore_secrets.py diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index af35d584..d8876f55 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,14 +1,209 @@ -name: Android builds +name: Android Builds on: + schedule: + - cron: "0 9 * * *" # 9am UTC = 1am PST / 2am PDT. for all testapps except firestore + pull_request: + types: [opened, reopened, synchronize] workflow_dispatch: inputs: apis: - description: 'CSV of apis whose quickstart examples we should build' + description: 'CSV of apis to build and test' + default: 'admob,analytics,auth,database,dynamic_links,firestore,functions,gma,messaging,remote_config,storage' + required: true + +env: + CCACHE_DIR: ${{ github.workspace }}/ccache_dir + GITHUB_TOKEN: ${{ github.token }} + xcodeVersion: "13.3.1" # Only affects Mac runners, and only for prerequisites. + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true jobs: - build: + check_and_prepare: runs-on: ubuntu-latest + outputs: + apis: ${{ steps.set_outputs.outputs.apis }} steps: - - name: noop - run: true + - id: set_outputs + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + apis="${{ github.event.inputs.apis }}" + else + apis="admob,analytics,auth,database,dynamic_links,firestore,functions,gma,messaging,remote_config,storage" + fi + echo apis: ${apis} + echo "::set-output name=apis::${apis}" + + build: + name: android-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.python_version }} + runs-on: ${{ matrix.os }} + needs: [check_and_prepare] + strategy: + fail-fast: false + matrix: + os: [macos-12, ubuntu-latest, windows-latest] + architecture: [x64] + python_version: [3.7] + steps: + - name: setup Xcode version (macos) + if: runner.os == 'macOS' + run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer + + - name: Store git credentials for all git commands + # Forces all git commands to use authenticated https, to prevent throttling. + shell: bash + run: | + git config --global credential.helper 'store --file /tmp/git-credentials' + echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials + + - name: Enable Git Long-paths Support + if: runner.os == 'Windows' + run: git config --system core.longpaths true + + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Set env variables for subsequent steps (all) + shell: bash + run: | + echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.architecture }}" >> $GITHUB_ENV + echo "GHA_INSTALL_CCACHE=1" >> $GITHUB_ENV + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + architecture: ${{ matrix.architecture }} + + - name: Add msbuild to PATH + if: startsWith(matrix.os, 'windows') + uses: microsoft/setup-msbuild@v1.0.2 + + - name: Cache NDK + id: cache_ndk + uses: actions/cache@v2 + with: + path: /tmp/android-ndk-r21e + key: android-ndk-${{ matrix.os }}-r21e + + - name: Check cached NDK + shell: bash + if: steps.cache_ndk.outputs.cache-hit != 'true' + run: | + # If the NDK failed to download from the cache, but there is a + # /tmp/android-ndk-r21e directory, it's incomplete, so remove it. + if [[ -d "/tmp/android-ndk-r21e" ]]; then + echo "Removing incomplete download of NDK" + rm -rf /tmp/android-ndk-r21e + fi + + - name: Update homebrew (avoid bintray errors) + uses: nick-invision/retry@v2 + if: startsWith(matrix.os, 'macos') + with: + timeout_minutes: 10 + max_attempts: 3 + command: | + # Temporarily here until Github runners have updated their version of + # homebrew. This prevents errors arising from the shut down of + # binutils, used by older version of homebrew for hosting packages. + brew update + + - name: Install prerequisites + uses: nick-invision/retry@v2 + with: + timeout_minutes: 10 + max_attempts: 3 + shell: bash + command: | + scripts/build_scripts/android/install_prereqs.sh + echo "NDK_ROOT=/tmp/android-ndk-r21e" >> $GITHUB_ENV + echo "ANDROID_NDK_HOME=/tmp/android-ndk-r21e" >> $GITHUB_ENV + pip install -r scripts/build_scripts/python_requirements.txt + python scripts/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" + + - name: Download C++ SDK + shell: bash + run: | + set +e + # Retry up to 10 times because Curl has a tendency to timeout on + # Github runners. + for retry in {1..10} error; do + if [[ $retry == "error" ]]; then exit 5; fi + curl -LSs \ + "https://firebase.google.com/download/cpp" \ + --output firebase_cpp_sdk.zip && break + sleep 300 + done + set -e + mkdir /tmp/downloaded_sdk + unzip -q firebase_cpp_sdk.zip -d /tmp/downloaded_sdk/ + rm -f firebase_cpp_sdk.zip + + - name: Cache ccache files + id: cache_ccache + uses: actions/cache@v2 + with: + path: ccache_dir + key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }} + + - name: Build testapp + shell: bash + run: | + set -x + python scripts/build_scripts/build_testapps.py --p Android \ + --t ${{ needs.check_and_prepare.outputs.apis }} \ + --output_directory "${{ github.workspace }}" \ + --artifact_name "android-${{ matrix.os }}" \ + --noadd_timestamp \ + --short_output_paths \ + --gha_build \ + --packaged_sdk /tmp/downloaded_sdk/firebase_cpp_sdk + + - name: Stats for ccache (mac and linux) + if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos') + run: ccache -s + + - name: Prepare results summary artifact + if: ${{ !cancelled() }} + shell: bash + run: | + if [ ! -f build-results-android-${{ matrix.os }}.log.json ]; then + echo "__SUMMARY_MISSING__" > build-results-android-${{ matrix.os }}.log.json + fi + + - name: Upload Android integration tests artifact + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: testapps-android-${{ matrix.os }} + path: testapps-android-${{ matrix.os }} + retention-days: ${{ env.artifactRetentionDays }} + + - name: Upload Android build results artifact + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: log-artifact + path: build-results-android-${{ matrix.os }}* + retention-days: ${{ env.artifactRetentionDays }} + + - name: Download log artifacts + if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }} + uses: actions/download-artifact@v3 + with: + path: test_results + name: log-artifact + + - name: Summarize build results + if: ${{ !cancelled() }} + shell: bash + run: | + cat build-results-android-${{ matrix.os }}.log + if [[ "${{ job.status }}" != "success" ]]; then + exit 1 + fi diff --git a/analytics/testapp/build.gradle b/analytics/testapp/build.gradle index 6928009e..7b5d686e 100644 --- a/analytics/testapp/build.gradle +++ b/analytics/testapp/build.gradle @@ -6,7 +6,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.2.1' + classpath 'com.android.tools.build:gradle:3.3.3' classpath 'com.google.gms:google-services:4.0.1' } } diff --git a/dynamic_links/testapp/build.gradle b/dynamic_links/testapp/build.gradle index bf4eff96..49bcf71b 100644 --- a/dynamic_links/testapp/build.gradle +++ b/dynamic_links/testapp/build.gradle @@ -6,7 +6,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.2.1' + classpath 'com.android.tools.build:gradle:3.3.3' classpath 'com.google.gms:google-services:4.0.1' } } diff --git a/scripts/build_scripts/android/install_prereqs.sh b/scripts/build_scripts/android/install_prereqs.sh new file mode 100755 index 00000000..f5d6efe9 --- /dev/null +++ b/scripts/build_scripts/android/install_prereqs.sh @@ -0,0 +1,75 @@ +#!/bin/bash -e + +# Copyright 2022 Google LLC + +if [[ $(uname) == "Darwin" ]]; then + platform=darwin + if [[ ! -z "${GHA_INSTALL_CCACHE}" ]]; then + brew install ccache + echo "CCACHE_INSTALLED=1" >> $GITHUB_ENV + fi +elif [[ $(uname) == "Linux" ]]; then + platform=linux + if [[ ! -z "${GHA_INSTALL_CCACHE}" ]]; then + sudo apt install ccache + echo "CCACHE_INSTALLED=1" >> $GITHUB_ENV + fi +else + platform=windows +fi + +if [[ -z $(which cmake) ]]; then + echo "Error, cmake is not installed or is not in the PATH." + exit 1 +fi + +if [[ -z $(which python) ]]; then + echo "Error, python is not installed or is not in the PATH." + exit 1 +else + updated_pip=0 + if ! $(echo "import absl"$'\n' | python - 2> /dev/null); then + echo "Installing python packages." + set -x + # On Windows bash shell, sudo doesn't exist + if [[ $(uname) == "Linux" ]] || [[ $(uname) == "Darwin" ]]; then + sudo python -m pip install --upgrade pip + else + python -m pip install --upgrade pip + fi + pip install absl-py + set +x + fi +fi + +if [[ -z "${ANDROID_HOME}" ]]; then + echo "Error, ANDROID_HOME environment variable is not set." + exit 1 +fi + +if [[ -z "${NDK_ROOT}" || -z $(grep "Pkg\.Revision = 21\." "${NDK_ROOT}/source.properties") ]]; then + if [[ -d /tmp/android-ndk-r21e && \ + -n $(grep "Pkg\.Revision = 21\." "/tmp/android-ndk-r21e/source.properties") ]]; then + echo "Using NDK r21e in /tmp/android-ndk-r21e". + else + echo "NDK_ROOT environment variable is not set, or NDK version is incorrect." + echo "This build recommends NDK r21e, downloading..." + if [[ -z $(which curl) ]]; then + echo "Error, could not run 'curl' to download NDK. Is it in your PATH?" + exit 1 + fi + set +e + # Retry up to 10 times because Curl has a tendency to timeout on + # Github runners. + for retry in {1..10} error; do + if [[ $retry == "error" ]]; then exit 5; fi + curl --http1.1 -LSs \ + "https://dl.google.com/android/repository/android-ndk-r21e-${platform}-x86_64.zip" \ + --output /tmp/android-ndk-r21e.zip && break + sleep 300 + done + set -e + (cd /tmp && unzip -oq android-ndk-r21e.zip && rm -f android-ndk-r21e.zip) + echo "NDK r21e has been downloaded into /tmp/android-ndk-r21e" + fi +fi diff --git a/scripts/build_scripts/build_testapps.json b/scripts/build_scripts/build_testapps.json new file mode 100644 index 00000000..7dc1d078 --- /dev/null +++ b/scripts/build_scripts/build_testapps.json @@ -0,0 +1,183 @@ +{ + "apis": [ + { + "name": "admob", + "full_name": "FirebaseAdmob", + "bundle_id": "com.google.ios.admob.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "admob/testapp", + "frameworks": [ + "firebase_admob.xcframework", + "firebase.xcframework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "analytics", + "full_name": "FirebaseAnalytics", + "bundle_id": "com.google.ios.analytics.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "analytics/testapp", + "frameworks": [ + "firebase_analytics.xcframework", + "firebase.xcframework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "auth", + "full_name": "FirebaseAuth", + "bundle_id": "com.google.FirebaseCppAuthTestApp.dev", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "auth/testapp", + "frameworks": [ + "firebase_auth.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Cpp_Auth_Test_App_Dev.mobileprovision" + }, + { + "name": "database", + "full_name": "FirebaseDatabase", + "bundle_id": "com.google.firebase.cpp.database.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "database/testapp", + "frameworks": [ + "firebase_auth.xcframework", + "firebase_database.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "dynamic_links", + "full_name": "FirebaseDynamicLinks", + "bundle_id": "com.google.FirebaseCppDynamicLinksTestApp.dev", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "dynamic_links/testapp", + "frameworks": [ + "firebase_dynamic_links.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Cpp_Dynamic_Links_Test_App_Dev.mobileprovision" + }, + { + "name": "functions", + "full_name": "FirebaseFunctions", + "bundle_id": "com.google.firebase.cpp.functions.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "functions/testapp", + "frameworks": [ + "firebase_auth.xcframework", + "firebase_functions.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "gma", + "full_name": "FirebaseGma", + "bundle_id": "com.google.ios.admob.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "gma/testapp", + "frameworks": [ + "firebase_gma.xcframework", + "firebase.xcframework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "messaging", + "full_name": "FirebaseMessaging", + "bundle_id": "com.google.FirebaseCppMessagingTestApp.dev", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "messaging/testapp", + "frameworks": [ + "firebase_messaging.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Cpp_Messaging_Test_App_Dev.mobileprovision" + }, + { + "name": "remote_config", + "full_name": "FirebaseRemoteConfig", + "bundle_id": "com.google.ios.remoteconfig.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "remote_config/testapp", + "frameworks": [ + "firebase_remote_config.xcframework", + "firebase.xcframework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "storage", + "full_name": "FirebaseStorage", + "bundle_id": "com.google.firebase.cpp.storage.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "storage/testapp", + "frameworks": [ + "firebase_storage.xcframework", + "firebase_auth.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision" + }, + { + "name": "firestore", + "full_name": "FirebaseFirestore", + "bundle_id": "com.google.firebase.cpp.firestore.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "firestore/testapp", + "frameworks": [ + "firebase_firestore.xcframework", + "firebase_auth.xcframework", + "firebase.xcframework" + ], + "provision": "Firebase_Dev_Wildcard.mobileprovision", + "minify": "proguard" + } + ], + "apple_team_id": "REPLACE_ME_TEMP_INVALID_ID", + "compiler_dict": { + "gcc-4.8": [ + "-DCMAKE_C_COMPILER=gcc-4.8", + "-DCMAKE_CXX_COMPILER=g++-4.8" + ], + "gcc-7": [ + "-DCMAKE_C_COMPILER=gcc-7", + "-DCMAKE_CXX_COMPILER=g++-7" + ], + "gcc-9": [ + "-DCMAKE_C_COMPILER=gcc-9", + "-DCMAKE_CXX_COMPILER=g++-9" + ], + "clang-5.0": [ + "-DCMAKE_C_COMPILER=clang-5.0", + "-DCMAKE_CXX_COMPILER=clang++-5.0" + ], + "VisualStudio2015": [ + "-G", + "Visual Studio 14 2015 Win64" + ], + "VisualStudio2017": [ + "-G", + "Visual Studio 15 2017 Win64" + ], + "VisualStudio2019": [ + "-G", + "Visual Studio 16 2019" + ] + } + } diff --git a/scripts/build_scripts/build_testapps.py b/scripts/build_scripts/build_testapps.py new file mode 100644 index 00000000..3caf85f0 --- /dev/null +++ b/scripts/build_scripts/build_testapps.py @@ -0,0 +1,718 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Build automation tool for Firebase C++ testapps for desktop and mobile. + +USAGE: + +This tool has a number of dependencies (listed below). Once those are taken +care of, here is an example of an execution of the tool (on MacOS): + +python build_testapps.py --t auth,messaging --p iOS --s /tmp/firebase-cpp-sdk + +Critical flags: +--t (full name: testapps, default: None) +--p (full name: platforms, default: None) +--s (full name: packaged_sdk, default: None) + +By default, this tool will build integration tests from source, which involves + +Under most circumstances the other flags don't need to be set, but can be +seen by running --help. Note that all path flags will forcefully expand +the user ~. + + +DEPENDENCIES: + +----Firebase Repo---- +The Firebase C++ Quickstart repo must be locally present. +Path specified by the flag: + + --repo_dir (default: current working directory) + +----Python Dependencies---- +The requirements.txt file has the required dependencies for this Python tool. + + pip install -r requirements.txt + +----CMake (Desktop only)---- +CMake must be installed and on the system path. + +----Environment Variables (Android only)---- +If building for Android, gradle requires several environment variables. +The following lists expected variables, and examples of what +a configured value may look like on MacOS: + + JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-8-latest/Contents/Home + ANDROID_HOME=/Users/user_name/Library/Android/sdk + ANDROID_SDK_HOME=/Users/user_name/Library/Android/sdk + ANDROID_NDK_HOME=/Users/user_name/Library/Android/sdk/ndk-bundle + +Or on Linux: + JAVA_HOME=/usr/local/buildtools/java/jdk/ + ANDROID_HOME=~/Android/Sdk + ANDROID_SDK_HOME=~/Android/Sdk + ANDROID_NDK_HOME=~/Android/Sdk/ndk + +If using this tool frequently, you will likely find it convenient to +modify your bashrc file to automatically set these variables. + +""" + +import attr +import datetime +import json +import os +import platform +import shutil +import stat +import subprocess +import sys +import tempfile + +from absl import app +from absl import flags +from absl import logging +from distutils import dir_util + +import utils +import config_reader +import xcodebuild + +# Environment variables +_JAVA_HOME = "JAVA_HOME" +_ANDROID_HOME = "ANDROID_HOME" +_ANDROID_SDK_HOME = "ANDROID_SDK_HOME" +_NDK_ROOT = "NDK_ROOT" +_ANDROID_NDK_HOME = "ANDROID_NDK_HOME" + +# Platforms +_ANDROID = "Android" +_IOS = "iOS" +_TVOS = "tvOS" +_DESKTOP = "Desktop" +_SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _TVOS, _DESKTOP) + +# Architecture +_SUPPORTED_ARCHITECTURES = ("x64", "x86", "arm64") + +# Values for iOS SDK flag (where the iOS app will run) +_APPLE_SDK_DEVICE = "real" +_APPLE_SDK_SIMULATOR = "virtual" +_SUPPORTED_APPLE_SDK = (_APPLE_SDK_DEVICE, _APPLE_SDK_SIMULATOR) + +_DEFAULT_RUN_TIMEOUT_SECONDS = 4800 # 1 hour 20 min + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "packaged_sdk", None, "Firebase SDK directory.") + +flags.DEFINE_string( + "output_directory", "~", + "Build output will be placed in this directory.") + +flags.DEFINE_string( + "artifact_name", "local-build", + "artifacts will be created and placed in output_directory." + " testapps artifact is testapps-$artifact_name;" + " build log artifact is build-results-$artifact_name.log.") + +flags.DEFINE_string( + "repo_dir", os.getcwd(), + "Firebase C++ Quickstart Git repository. Current directory by default.") + +flags.DEFINE_list( + "testapps", None, "Which testapps (Firebase APIs) to build, e.g." + " 'analytics,auth'.", + short_name="t") + +flags.DEFINE_list( + "platforms", None, "Which platforms to build. Can be Android, iOS and/or" + " Desktop", short_name="p") + +flags.DEFINE_bool( + "add_timestamp", True, + "Add a timestamp to the output directory for disambiguation." + " Recommended when running locally, so each execution gets its own " + " directory.") + +flags.DEFINE_list( + "ios_sdk", _APPLE_SDK_DEVICE, + "(iOS only) Build for real device (.ipa), virtual device / simulator (.app), " + "or both. Building for both will produce both an .app and an .ipa.") + +flags.DEFINE_list( + "tvos_sdk", _APPLE_SDK_SIMULATOR, + "(tvOS only) Build for real device (.ipa), virtual device / simulator (.app), " + "or both. Building for both will produce both an .app and an .ipa.") + +flags.DEFINE_bool( + "update_pod_repo", True, + "(iOS/tvOS only) Will run 'pod repo update' before building for iOS/tvOS to update" + " the local spec repos available on this machine. Must also include iOS/tvOS" + " in platforms flag.") + +flags.DEFINE_string( + "compiler", None, + "(Desktop only) Specify the compiler with CMake during the testapps build." + " Check the config file to see valid choices for this flag." + " If none, will invoke cmake without specifying a compiler.") + +flags.DEFINE_string( + "arch", "x64", + "(Desktop only) Which architecture to build: x64 (all), x86 (Windows/Linux), " + "or arm64 (Mac only).") + +# Get the number of CPUs for the default value of FLAGS.jobs +CPU_COUNT = os.cpu_count(); +# If CPU count couldn't be determined, default to 2. +DEFAULT_CPU_COUNT = 2 +if CPU_COUNT is None: CPU_COUNT = DEFAULT_CPU_COUNT +# Cap at 4 CPUs. +MAX_CPU_COUNT = 4 +if CPU_COUNT > MAX_CPU_COUNT: CPU_COUNT = MAX_CPU_COUNT + +flags.DEFINE_integer( + "jobs", CPU_COUNT, + "(Desktop only) If > 0, pass in -j to make CMake parallelize the" + " build. Defaults to the system's CPU count (max %s)." % MAX_CPU_COUNT) + +flags.DEFINE_multi_string( + "cmake_flag", None, + "Pass an additional flag to the CMake configure step." + " This option can be specified multiple times.") + +flags.register_validator( + "platforms", + lambda p: all(platform in _SUPPORTED_PLATFORMS for platform in p), + message="Valid platforms: " + ",".join(_SUPPORTED_PLATFORMS), + flag_values=FLAGS) + +flags.register_validator( + "ios_sdk", + lambda s: all(ios_sdk in _SUPPORTED_APPLE_SDK for ios_sdk in s), + message="Valid platforms: " + ",".join(_SUPPORTED_APPLE_SDK), + flag_values=FLAGS) + +flags.register_validator( + "tvos_sdk", + lambda s: all(tvos_sdk in _SUPPORTED_APPLE_SDK for tvos_sdk in s), + message="Valid platforms: " + ",".join(_SUPPORTED_APPLE_SDK), + flag_values=FLAGS) + +flags.DEFINE_bool( + "short_output_paths", False, + "Use short directory names for output paths. Useful to avoid hitting file " + "path limits on Windows.") + +flags.DEFINE_bool( + "gha_build", False, + "Set to true if this is a GitHub Actions build.") + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + platforms = FLAGS.platforms + testapps = FLAGS.testapps + + sdk_dir = _fix_path(FLAGS.packaged_sdk) + root_output_dir = _fix_path(FLAGS.output_directory) + repo_dir = _fix_path(FLAGS.repo_dir) + + update_pod_repo = FLAGS.update_pod_repo + if FLAGS.add_timestamp: + timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + else: + timestamp = "" + + if FLAGS.short_output_paths: + output_dir = os.path.join(root_output_dir, "ta") + else: + output_dir = os.path.join(root_output_dir, "testapps" + timestamp) + + config = config_reader.read_config() + + xcframework_dir = os.path.join(sdk_dir, "xcframeworks") + xcframework_exist = os.path.isdir(xcframework_dir) + if not xcframework_exist: + if _IOS in platforms: + _build_xcframework_from_repo(repo_dir, "ios", testapps, config) + if _TVOS in platforms: + _build_xcframework_from_repo(repo_dir, "tvos", testapps, config) + + if update_pod_repo and (_IOS in platforms or _TVOS in platforms): + _run(["pod", "repo", "update"]) + + cmake_flags = _get_desktop_compiler_flags(FLAGS.compiler, config.compilers) + + if (_DESKTOP in platforms and utils.is_linux_os() and FLAGS.arch == "x86"): + # Write out a temporary toolchain file to force 32-bit Linux builds, as + # the SDK-included toolchain file may not be present when building against + # the packaged SDK. + temp_toolchain_file = tempfile.NamedTemporaryFile("w+", suffix=".cmake") + temp_toolchain_file.writelines([ + 'set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")\n', + 'set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")\n', + 'set(CMAKE_LIBRARY_PATH "/usr/lib/i386-linux-gnu")\n', + 'set(INCLUDE_DIRECTORIES ${INCLUDE_DIRECTORIES} "/usr/include/i386-linux-gnu")\n']) + temp_toolchain_file.flush() + # Leave the file open, as it will be deleted on close, i.e. when this script exits. + # (On Linux, the file can be opened a second time by cmake while still open by + # this script) + cmake_flags.extend(["-DCMAKE_TOOLCHAIN_FILE=%s" % temp_toolchain_file.name]) + + if FLAGS.cmake_flag: + cmake_flags.extend(FLAGS.cmake_flag) + + failures = [] + for testapp in testapps: + api_config = config.get_api(testapp) + testapp_dirs = [api_config.testapp_path] + for testapp_dir in testapp_dirs: + logging.info("BEGIN building for %s: %s", testapp, testapp_dir) + failures += _build( + testapp=testapp, + platforms=platforms, + api_config=config.get_api(testapp), + testapp_dir=testapp_dir, + output_dir=output_dir, + sdk_dir=sdk_dir, + xcframework_exist=xcframework_exist, + repo_dir=repo_dir, + ios_sdk=FLAGS.ios_sdk, + tvos_sdk=FLAGS.tvos_sdk, + cmake_flags=cmake_flags, + short_output_paths=FLAGS.short_output_paths) + logging.info("END building for %s", testapp) + + _collect_integration_tests(testapps, root_output_dir, output_dir, FLAGS.artifact_name) + + _summarize_results(testapps, platforms, failures, root_output_dir, FLAGS.artifact_name) + return 1 if failures else 0 + + +def _build( + testapp, platforms, api_config, testapp_dir, output_dir, sdk_dir, xcframework_exist, + repo_dir, ios_sdk, tvos_sdk, cmake_flags, short_output_paths): + """Builds one testapp on each of the specified platforms.""" + os.chdir(repo_dir) + project_dir = os.path.join(output_dir, api_config.name) + if short_output_paths: + # Combining the first letter of every part separated by underscore for + # testapp paths. This is a trick to reduce file path length as we were + # exceeding the limit on Windows. + testapp_dir_parts = os.path.basename(testapp_dir).split('_') + output_testapp_dir = ''.join([x[0] for x in testapp_dir_parts]) + else: + output_testapp_dir = os.path.basename(testapp_dir) + + project_dir = os.path.join(project_dir, output_testapp_dir) + + logging.info("Copying testapp project to %s", project_dir) + os.makedirs(project_dir) + dir_util.copy_tree(testapp_dir, project_dir) + + logging.info("Changing directory to %s", project_dir) + os.chdir(project_dir) + + # TODO(DDB): remove + # _run_setup_script(repo_dir, project_dir) + + failures = [] + + if _DESKTOP in platforms: + logging.info("BEGIN %s, %s", testapp, _DESKTOP) + try: + _build_desktop(sdk_dir, cmake_flags) + except subprocess.SubprocessError as e: + failures.append( + Failure(testapp=testapp, platform=_DESKTOP, error_message=str(e))) + _rm_dir_safe(os.path.join(project_dir, "bin")) + logging.info("END %s, %s", testapp, _DESKTOP) + + if _ANDROID in platforms: + logging.info("BEGIN %s, %s", testapp, _ANDROID) + try: + _validate_android_environment_variables() + _build_android(project_dir, sdk_dir) + except subprocess.SubprocessError as e: + failures.append( + Failure(testapp=testapp, platform=_ANDROID, error_message=str(e))) + _rm_dir_safe(os.path.join(project_dir, "build", "intermediates")) + _rm_dir_safe(os.path.join(project_dir, ".externalNativeBuild")) + logging.info("END %s, %s", testapp, _ANDROID) + + if _IOS in platforms: + logging.info("BEGIN %s, %s", testapp, _IOS) + try: + _build_apple( + sdk_dir=sdk_dir, + xcframework_exist=xcframework_exist, + project_dir=project_dir, + repo_dir=repo_dir, + api_config=api_config, + target=api_config.ios_target, + scheme=api_config.ios_scheme, + apple_platfrom=_IOS, + apple_sdk=ios_sdk) + + except subprocess.SubprocessError as e: + failures.append( + Failure(testapp=testapp, platform=_IOS, error_message=str(e))) + logging.info("END %s, %s", testapp, _IOS) + + if _TVOS in platforms and api_config.tvos_target: + logging.info("BEGIN %s, %s", testapp, _TVOS) + try: + _build_apple( + sdk_dir=sdk_dir, + xcframework_exist=xcframework_exist, + project_dir=project_dir, + repo_dir=repo_dir, + api_config=api_config, + target=api_config.tvos_target, + scheme=api_config.tvos_scheme, + apple_platfrom=_TVOS, + apple_sdk=tvos_sdk) + except subprocess.SubprocessError as e: + failures.append( + Failure(testapp=testapp, platform=_TVOS, error_message=str(e))) + logging.info("END %s, %s", testapp, _TVOS) + + return failures + + +def _collect_integration_tests(testapps, root_output_dir, output_dir, artifact_name): + testapps_artifact_dir = "testapps-" + artifact_name + android_testapp_extension = ".apk" + ios_testapp_extension = ".ipa" + ios_simualtor_testapp_extension = ".app" + desktop_testapp_name = "testapp" + if platform.system() == "Windows": + desktop_testapp_name += ".exe" + + testapp_paths = [] + testapp_google_services = {} + for file_dir, directories, file_names in os.walk(output_dir): + for directory in directories: + if directory.endswith(ios_simualtor_testapp_extension): + testapp_paths.append(os.path.join(file_dir, directory)) + for file_name in file_names: + if ((file_name == desktop_testapp_name and "ios_build" not in file_dir) + or file_name.endswith(android_testapp_extension) + or file_name.endswith(ios_testapp_extension)): + testapp_paths.append(os.path.join(file_dir, file_name)) + if (file_name == "google-services.json"): + testapp_google_services[file_dir.split(os.path.sep)[-2]] = os.path.join(file_dir, file_name) + + artifact_path = os.path.join(root_output_dir, testapps_artifact_dir) + _rm_dir_safe(artifact_path) + for testapp in testapps: + os.makedirs(os.path.join(artifact_path, testapp)) + for path in testapp_paths: + for testapp in testapps: + if testapp in path: + if os.path.isfile(path): + shutil.copy(path, os.path.join(artifact_path, testapp)) + if path.endswith(desktop_testapp_name) and testapp_google_services.get(testapp): + shutil.copy(testapp_google_services[testapp], os.path.join(artifact_path, testapp)) + else: + dir_util.copy_tree(path, os.path.join(artifact_path, testapp, os.path.basename(path))) + break + + +def _write_summary(testapp_dir, summary, file_name="summary.log"): + with open(os.path.join(testapp_dir, file_name), "a") as f: + timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") + f.write("\n%s\n%s\n" % (timestamp, summary)) + + +def _summarize_results(testapps, platforms, failures, root_output_dir, artifact_name): + """Logs a readable summary of the results of the build.""" + file_name = "build-results-" + artifact_name + ".log" + + summary = [] + summary.append("BUILD SUMMARY:") + summary.append("TRIED TO BUILD: " + ",".join(testapps)) + summary.append("ON PLATFORMS: " + ",".join(platforms)) + + if not failures: + summary.append("ALL BUILDS SUCCEEDED") + else: + summary.append("SOME ERRORS OCCURRED:") + for i, failure in enumerate(failures, start=1): + summary.append("%d: %s" % (i, failure.describe())) + summary = "\n".join(summary) + + logging.info(summary) + _write_summary(root_output_dir, summary, file_name=file_name) + + summary_json = {} + summary_json["type"] = "build" + summary_json["testapps"] = testapps + summary_json["errors"] = {failure.testapp:failure.error_message for failure in failures} + with open(os.path.join(root_output_dir, file_name+".json"), "a") as f: + f.write(json.dumps(summary_json, indent=2)) + + +def _build_desktop(sdk_dir, cmake_flags): + cmake_configure_cmd = ["cmake", ".", "-DCMAKE_BUILD_TYPE=Debug", + "-DFIREBASE_CPP_SDK_DIR=" + sdk_dir] + if utils.is_windows_os(): + cmake_configure_cmd += ["-A", + "Win32" if FLAGS.arch == "x86" else FLAGS.arch] + elif utils.is_mac_os(): + # Ensure that correct Mac architecture is built. + cmake_configure_cmd += ["-DCMAKE_OSX_ARCHITECTURES=%s" % + ("arm64" if FLAGS.arch == "arm64" else "x86_64")] + + _run(cmake_configure_cmd + cmake_flags) + _run(["cmake", "--build", ".", "--config", "Debug"] + + ["-j", str(FLAGS.jobs)] if FLAGS.jobs > 0 else []) + + +def _get_desktop_compiler_flags(compiler, compiler_table): + """Returns the command line flags for this compiler.""" + if not compiler: # None is an acceptable default value + return [] + try: + return compiler_table[compiler] + except KeyError: + valid_keys = ", ".join(compiler_table.keys()) + raise ValueError( + "Given compiler: %s. Valid compilers: %s" % (compiler, valid_keys)) + + +def _build_android(project_dir, sdk_dir): + """Builds an Android binary (apk).""" + if platform.system() == "Windows": + gradlew = "gradlew.bat" + sdk_dir = sdk_dir.replace("\\", "/") # Gradle misinterprets backslashes. + else: + gradlew = "./gradlew" + logging.info("Patching gradle properties with path to SDK") + gradle_properties = os.path.join(project_dir, "gradle.properties") + with open(gradle_properties, "a+") as f: + f.write("systemProp.firebase_cpp_sdk.dir=" + sdk_dir + "\n") + f.write("http.keepAlive=false\n") + f.write("maven.wagon.http.pool=false\n") + f.write("maven.wagon.httpconnectionManager.ttlSeconds=120") + # This will log the versions of dependencies for debugging purposes. + _run([gradlew, "dependencies", "--configuration", "debugCompileClasspath",]) + _run([gradlew, "assembleDebug", "--stacktrace"]) + + +def _validate_android_environment_variables(): + """Checks environment variables that may be required for Android.""" + # Ultimately we let the gradle build be the source of truth on what env vars + # are required, but try to repair holes and log warnings if we can't. + android_home = os.environ.get(_ANDROID_HOME) + if not os.environ.get(_JAVA_HOME): + logging.warning("%s not set", _JAVA_HOME) + if not os.environ.get(_ANDROID_SDK_HOME): + if android_home: # Use ANDROID_HOME as backup for ANDROID_SDK_HOME + os.environ[_ANDROID_SDK_HOME] = android_home + logging.info("%s not found, using %s", _ANDROID_SDK_HOME, _ANDROID_HOME) + else: + logging.warning("Missing: %s and %s", _ANDROID_SDK_HOME, _ANDROID_HOME) + # Different environments may have different NDK env vars specified. We look + # for these, in this order, and set the others to the first found. + # If none are set, we check the default location for the ndk. + ndk_path = None + ndk_vars = [_NDK_ROOT, _ANDROID_NDK_HOME] + for env_var in ndk_vars: + val = os.environ.get(env_var) + if val: + ndk_path = val + break + if not ndk_path: + if android_home: + default_ndk_path = os.path.join(android_home, "ndk-bundle") + if os.path.isdir(default_ndk_path): + ndk_path = default_ndk_path + if ndk_path: + logging.info("Found ndk: %s", ndk_path) + for env_var in ndk_vars: + if os.environ.get(env_var) != ndk_path: + logging.info("Setting %s to %s", env_var, ndk_path) + os.environ[env_var] = ndk_path + else: + logging.warning("No NDK env var set. Set one of %s", ", ".join(ndk_vars)) + +# build required ios xcframeworks based on makefiles +# the xcframeworks locates at repo_dir/ios_build +def _build_xcframework_from_repo(repo_dir, apple_platform, testapps, config): + """Builds xcframework from SDK source.""" + output_path = os.path.join(repo_dir, apple_platform + "_build") + _rm_dir_safe(output_path) + xcframework_builder = os.path.join( + repo_dir, "scripts", "gha", "build_ios_tvos.py") + + # build only required targets to save time + target = set() + for testapp in testapps: + api_config = config.get_api(testapp) + if apple_platform == "ios" or (apple_platform == "tvos" and api_config.tvos_target): + for framework in api_config.frameworks: + # firebase_analytics.framework -> firebase_analytics + target.add(os.path.splitext(framework)[0]) + + # firebase is not a target in CMake, firebase_app is the target + # firebase_app will be built by other target as well + target.remove("firebase") + + framework_builder_args = [ + sys.executable, xcframework_builder, + "-b", output_path, + "-s", repo_dir, + "-o", apple_platform, + "-t" + ] + framework_builder_args.extend(target) + _run(framework_builder_args) + + +def _build_apple( + sdk_dir, xcframework_exist, project_dir, repo_dir, api_config, + target, scheme, apple_platfrom, apple_sdk): + """Builds an iOS application (.app, .ipa or both).""" + build_dir = apple_platfrom.lower() + "_build" + if not xcframework_exist: + sdk_dir = os.path.join(repo_dir, build_dir) + + build_dir = os.path.join(project_dir, build_dir) + os.makedirs(build_dir) + + logging.info("Copying XCFrameworks") + framework_src_dir = os.path.join(sdk_dir, "xcframeworks") + framework_paths = [] # Paths to the copied frameworks. + for framework in api_config.frameworks: + framework_src_path = os.path.join(framework_src_dir, framework) + framework_dest_path = os.path.join(project_dir, "Frameworks", framework) + dir_util.copy_tree(framework_src_path, framework_dest_path) + framework_paths.append(framework_dest_path) + + _run(["pod", "install"]) + + entitlements_path = os.path.join( + project_dir, api_config.ios_target + ".entitlements") + xcode_tool_path = os.path.join( + repo_dir, "scripts", "gha", "integration_testing", "xcode_tool.rb") + xcode_patcher_args = [ + "ruby", xcode_tool_path, + "--XCodeCPP.xcodeProjectDir", project_dir, + "--XCodeCPP.target", target, + "--XCodeCPP.frameworks", ",".join(framework_paths) + ] + # Internal integration tests require the SDK root as an include path. + if repo_dir and api_config.internal_testapp_path: + xcode_patcher_args.extend(("--XCodeCPP.include", repo_dir)) + if os.path.isfile(entitlements_path): # Not all testapps require entitlements + logging.info("Entitlements file detected.") + xcode_patcher_args.extend(("--XCodeCPP.entitlement", entitlements_path)) + else: + logging.info("No entitlements found at %s.", entitlements_path) + _run(xcode_patcher_args) + + xcode_path = os.path.join(project_dir, "integration_test.xcworkspace") + if _APPLE_SDK_SIMULATOR in apple_sdk: + _run( + xcodebuild.get_args_for_build( + path=xcode_path, + scheme=scheme, + output_dir=build_dir, + apple_platfrom=apple_platfrom, + apple_sdk=_APPLE_SDK_SIMULATOR, + configuration="Debug")) + + if _APPLE_SDK_DEVICE in apple_sdk: + _run( + xcodebuild.get_args_for_build( + path=xcode_path, + scheme=scheme, + output_dir=build_dir, + apple_platfrom=apple_platfrom, + apple_sdk=_APPLE_SDK_DEVICE, + configuration="Debug")) + + xcodebuild.generate_unsigned_ipa( + output_dir=build_dir, configuration="Debug") + + +# This should be executed before performing any builds. +def _run_setup_script(root_dir, testapp_dir): + """Runs the setup_integration_tests.py script.""" + # This script will download gtest to its own directory. + # The CMake projects were configured to download gtest, but this was + # found to be flaky and errors didn't propagate up the build system + # layers. The workaround is to download gtest with this script and copy it. + downloader_dir = os.path.join(root_dir, "testing", "test_framework") + _run([sys.executable, os.path.join(downloader_dir, "download_googletest.py")]) + # Copies shared test framework files into the project, including gtest. + script_path = os.path.join(root_dir, "setup_integration_tests.py") + _run([sys.executable, script_path, testapp_dir]) + + +def _run(args, timeout=_DEFAULT_RUN_TIMEOUT_SECONDS, capture_output=False, text=None, check=True): + """Executes a command in a subprocess.""" + logging.info("Running in subprocess: %s", " ".join(args)) + return subprocess.run( + args=args, + timeout=timeout, + capture_output=capture_output, + text=text, + check=check) + + +def _handle_readonly_file(func, path, excinfo): + """Function passed into shutil.rmtree to handle Access Denied error""" + os.chmod(path, stat.S_IWRITE) + func(path) # will re-throw if a different error occurrs + + +def _rm_dir_safe(directory_path): + """Removes directory at given path. No error if dir doesn't exist.""" + logging.info("Deleting %s...", directory_path) + try: + shutil.rmtree(directory_path, onerror=_handle_readonly_file) + except OSError as e: + # There are two known cases where this can happen: + # The directory doesn't exist (FileNotFoundError) + # A file in the directory is open in another process (PermissionError) + logging.warning("Failed to remove directory:\n%s", e.strerror) + + +def _fix_path(path): + """Expands ~, normalizes slashes, and converts relative paths to absolute.""" + return os.path.abspath(os.path.expanduser(path)) + + +@attr.s(frozen=True, eq=False) +class Failure(object): + """Holds context for the failure of a testapp to build/run.""" + testapp = attr.ib() + platform = attr.ib() + error_message = attr.ib() + + def describe(self): + return "%s, %s: %s" % (self.testapp, self.platform, self.error_message) + + +if __name__ == "__main__": + flags.mark_flag_as_required("testapps") + flags.mark_flag_as_required("platforms") + flags.mark_flag_as_required("packaged_sdk") + app.run(main) diff --git a/scripts/build_scripts/config_reader.py b/scripts/build_scripts/config_reader.py new file mode 100644 index 00000000..a24f5d7a --- /dev/null +++ b/scripts/build_scripts/config_reader.py @@ -0,0 +1,139 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A utility for working with testapp builder JSON files. + +This module handles loading the central configuration file for a testapp +builder, returning a 'Config' object that exposes all the data. + +The motivation for loading the config into a class as opposed to returning +the loaded JSON directly is to validate the data upfront, to fail fast if +anything is missing or formatted incorrectly. + +Example of such a configuration file: + +{ + "apis": [ + { + "name": "analytics", + "full_name": "FirebaseAnalytics", + "bundle_id": "com.google.ios.analytics.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "analytics/testapp", + "frameworks": [ + "firebase_analytics.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + }, + { + "name": "admob", + "full_name": "FirebaseAdmob", + "bundle_id": "com.google.ios.admob.testapp", + "ios_target": "testapp", + "tvos_target": "", + "testapp_path": "admob/testapp", + "frameworks": [ + "firebase_admob.framework", + "firebase.framework" + ], + "provision": "Google_Development.mobileprovision" + } + ], + "dev_team": "ABCDEFGHIJK" +} + +""" + +import json +import os +import pathlib + +import attr + +_DEFAULT_CONFIG_NAME = "build_testapps.json" + + +def read_config(path=None): + """Creates an in-memory 'Config' object out of a testapp config file. + + Args: + path (str): Path to a testapp builder config file. If not specified, will + look for 'build_testapps.json' in the same directory as this file. + + Returns: + Config: All of the testapp builder's configuration. + + """ + if not path: + directory = pathlib.Path(__file__).parent.absolute() + path = os.path.join(directory, _DEFAULT_CONFIG_NAME) + with open(path, "r") as config: + config = json.load(config) + api_configs = dict() + try: + for api in config["apis"]: + api_name = api["name"] + api_configs[api_name] = APIConfig( + name=api_name, + full_name=api["full_name"], + bundle_id=api["bundle_id"], + ios_target=api["ios_target"], + tvos_target=api["tvos_target"], + ios_scheme=api["ios_target"], # Scheme assumed to be same as target. + tvos_scheme=api["tvos_target"], + testapp_path=api["testapp_path"], + internal_testapp_path=api.get("internal_testapp_path", None), + frameworks=api["frameworks"], + provision=api["provision"], + minify=api.get("minify", None)) + return Config( + apis=api_configs, + compilers=config["compiler_dict"]) + except (KeyError, TypeError, IndexError): + # The error will be cryptic on its own, so we dump the JSON to + # offer context, then reraise the error. + print( + "Error occurred while parsing config. Full config dump:\n" + + json.dumps(config, sort_keys=True, indent=4, separators=(",", ":"))) + raise + + +@attr.s(frozen=True, eq=False) +class Config(object): + apis = attr.ib() # Mapping of str: APIConfig + compilers = attr.ib() + + def get_api(self, api): + """Returns the APIConfig object for the given api, e.g. 'analytics'.""" + return self.apis[api] + + +@attr.s(frozen=True, eq=False) +class APIConfig(object): + """Holds all the configuration for a single testapp project.""" + name = attr.ib() + full_name = attr.ib() + bundle_id = attr.ib() + ios_target = attr.ib() + tvos_target = attr.ib() + ios_scheme = attr.ib() + tvos_scheme = attr.ib() + testapp_path = attr.ib() # testapp dir relative to sdk root + internal_testapp_path = attr.ib() # Internal testdir dir relative to sdk root + frameworks = attr.ib() # Required custom xcode frameworks + provision = attr.ib() # Path to the local mobile provision + minify = attr.ib() # (Optional) Android minification. + diff --git a/scripts/build_scripts/python_requirements.txt b/scripts/build_scripts/python_requirements.txt new file mode 100644 index 00000000..214a8aac --- /dev/null +++ b/scripts/build_scripts/python_requirements.txt @@ -0,0 +1,2 @@ +attrs +absl-py diff --git a/scripts/build_scripts/utils.py b/scripts/build_scripts/utils.py new file mode 100644 index 00000000..631dc8df --- /dev/null +++ b/scripts/build_scripts/utils.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +# Copyright 2022 Google +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Helper functions that are shared amongst prereqs and build scripts across various +platforms. +""" + +import distutils.spawn +import platform +import shutil +import subprocess +import os +import urllib.request + +def run_command(cmd, capture_output=False, cwd=None, check=False, as_root=False, + print_cmd=True): + """Run a command. + + Args: + cmd (list(str)): Command to run as a list object. + Eg: ['ls', '-l']. + capture_output (bool): Capture the output of this command. + Output can be accessed as .stdout + cwd (str): Directory to execute the command from. + check (bool): Raises a CalledProcessError if True and the command errored out + as_root (bool): Run command as root user with admin priveleges (supported on mac and linux). + print_cmd (bool): Print the command we are running to stdout. + + Raises: + (subprocess.CalledProcessError): If command errored out and `text=True` + + Returns: + (`subprocess.CompletedProcess`): object containing information from + command execution + """ + + if as_root and (is_mac_os() or is_linux_os()): + cmd.insert(0, 'sudo') + + cmd_string = ' '.join(cmd) + if print_cmd: + print('Running cmd: {0}\n'.format(cmd_string)) + # If capture_output is requested, we also set text=True to store the returned value of the + # command as a string instead of bytes object + return subprocess.run(cmd, capture_output=capture_output, cwd=cwd, + check=check, text=capture_output) + + +def is_command_installed(tool): + """Check if a command is installed on the system.""" + return distutils.spawn.find_executable(tool) + + +def delete_directory(dir_path): + """Recursively delete a valid directory""" + if os.path.exists(dir_path): + shutil.rmtree(dir_path) + + +def download_file(url, file_path): + """Download from url and save to specified file path.""" + with urllib.request.urlopen(url) as response, open(file_path, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + + +def unpack_files(archive_file_path, output_dir=None): + """Unpack/extract an archive to specified output_directory""" + shutil.unpack_archive(archive_file_path, output_dir) + + +def is_windows_os(): + return platform.system() == 'Windows' + + +def is_mac_os(): + return platform.system() == 'Darwin' + + +def is_linux_os(): + return platform.system() == 'Linux' diff --git a/scripts/build_scripts/xcodebuild.py b/scripts/build_scripts/xcodebuild.py new file mode 100644 index 00000000..b60c70dc --- /dev/null +++ b/scripts/build_scripts/xcodebuild.py @@ -0,0 +1,119 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper module for working with xcode projects. + +The tool xcodebuild provides support to build xcode projects from the command +line. The motivation was to simplify usage of xcodebuild, since it was non-trivial +to figure out which flags were needed to get it working in a CI environment. +The options required by the methods in this module were found to work both +locally and on CI, with both the Unity and C++ projects. + +get_args_for_build() method doesn't performing operations with xcodebuild directly, +this module returns arg sequences. These sequences can be passed to e.g. +subprocess.run to execute the operations. + +get_args_for_build() support either device or simulator builds. For simulator +builds, it suffices to use get_args_for_build() to create a .app that can be +used with simulators. For unsigned device builds, generate .app via +get_args_for_build() step and then use generate_unsigned_ipa() to package +the .app to .ipa. + +""" + +import os +import shutil + +def get_args_for_build( + path, scheme, output_dir, apple_platfrom, apple_sdk, configuration): + """Constructs subprocess args for an unsigned xcode build. + + Args: + path (str): Full path to the project or workspace to build. Must end in + either .xcodeproj or .xcworkspace. + scheme (str): Name of the scheme to build. + output_dir (str): Directory for the resulting build artifacts. Will be + created if it doesn't already exist. + apple_platfrom (str): iOS or tvOS. + apple_sdk (str): Where this build will be run: real device or virtual device (simulator). + configuration (str): Value for the -configuration flag. + + Returns: + Sequence of strings, corresponding to valid args for a subprocess call. + + """ + args = [ + "xcodebuild", + "-sdk", _get_apple_env_from_target(apple_platfrom, apple_sdk), + "-scheme", scheme, + "-configuration", configuration, + "-quiet", + "BUILD_DIR=" + output_dir + ] + + if apple_sdk == "real": + args.extend(['CODE_SIGN_IDENTITY=""', + "CODE_SIGNING_REQUIRED=NO", + "CODE_SIGNING_ALLOWED=NO"]) + elif apple_sdk == "virtual" and apple_platfrom == "tvOS": + args.extend(['-arch', "x86_64"]) + + if not path: + raise ValueError("Must supply a path.") + if path.endswith(".xcworkspace"): + args.extend(("-workspace", path)) + elif path.endswith(".xcodeproj"): + args.extend(("-project", path)) + else: + raise ValueError("Path must end with .xcworkspace or .xcodeproj: %s" % path) + return args + + +def _get_apple_env_from_target(apple_platfrom, apple_sdk): + """Return a value for the -sdk flag based on the target (device/simulator).""" + if apple_platfrom == "iOS": + if apple_sdk == "real": + return "iphoneos" + elif apple_sdk == "virtual": + return "iphonesimulator" + else: + raise ValueError("Unrecognized apple_sdk: %s" % apple_sdk) + elif apple_platfrom == "tvOS": + if apple_sdk == "real": + return "appletvos" + elif apple_sdk == "virtual": + return "appletvsimulator" + else: + raise ValueError("Unrecognized apple_sdk: %s" % apple_sdk) + else: + raise ValueError("Unrecognized apple_sdk: %s" % apple_sdk) + + +def generate_unsigned_ipa(output_dir, configuration): + """create unsigned .ipa from .app, then remove .app afterwards + + Args: + output_dir (str): Same value as get_args_for_build. generated unsigned .ipa + will be placed within the subdirectory "Debug-iphoneos" or "Release-iphoneos". + configuration (str): Same value as get_args_for_build. + """ + iphone_build_dir = os.path.join(output_dir, configuration + "-iphoneos") + payload_path = os.path.join(iphone_build_dir, "Payload") + app_path = os.path.join(iphone_build_dir, "integration_test.app") + ipa_path = os.path.join(iphone_build_dir, "integration_test.ipa") + os.mkdir(payload_path) + shutil.move(app_path, payload_path) + shutil.make_archive(payload_path, 'zip', root_dir=iphone_build_dir, base_dir='Payload') + shutil.move('%s.%s'%(payload_path, 'zip'), ipa_path) + shutil.rmtree(payload_path) diff --git a/scripts/restore_secrets.py b/scripts/restore_secrets.py new file mode 100644 index 00000000..cd72cf58 --- /dev/null +++ b/scripts/restore_secrets.py @@ -0,0 +1,186 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script for restoring secrets into the testapp projects. + +Usage: + +python restore_secrets.py --passphrase [--repo_dir ] +python restore_secrets.py --passphrase_file [--repo_dir ] + +--passphrase: Passphrase to decrypt the files. This option is insecure on a + multi-user machine; use the --passphrase_file option instead. +--passphrase_file: Specify a file to read the passphrase from (only reads the + first line). Use "-" (without quotes) for stdin. +--repo_dir: Path to C++ Quickstart Github repository. Defaults to current + directory. +--apis: Specify a list of particular product APIs and retrieve only their + secrets. + +This script will perform the following: + +- Google Service files (plist and json) will be restored into the + testapp directories. +- The reverse id will be patched into all Info.plist files, using the value from + the decrypted Google Service plist files as the source of truth. + +""" + +import os +import plistlib +import subprocess + +from absl import app +from absl import flags + + +FLAGS = flags.FLAGS + +flags.DEFINE_string("repo_dir", os.getcwd(), "Path to C++ SDK Github repo.") +flags.DEFINE_string("passphrase", None, "The passphrase itself.") +flags.DEFINE_string("passphrase_file", None, + "Path to file with passphrase. Use \"-\" (without quotes) for stdin.") +flags.DEFINE_string("artifact", None, "Artifact Path, google-services.json will be placed here.") +flags.DEFINE_list("apis",[], "Optional comma-separated list of APIs for which to retreive " + " secrets. All secrets will be fetched if this is flag is not defined.") + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + repo_dir = FLAGS.repo_dir + # The passphrase is sensitive, do not log. + if FLAGS.passphrase: + passphrase = FLAGS.passphrase + elif FLAGS.passphrase_file == "-": + passphrase = input() + elif FLAGS.passphrase_file: + with open(FLAGS.passphrase_file, "r") as f: + passphrase = f.readline().strip() + else: + raise ValueError("Must supply passphrase or passphrase_file arg.") + + if FLAGS.apis: + print("Retrieving secrets for product APIs: ", FLAGS.apis) + + secrets_dir = os.path.join(repo_dir, "scripts", "gha-encrypted") + encrypted_files = _find_encrypted_files(secrets_dir) + print("Found these encrypted files:\n%s" % "\n".join(encrypted_files)) + + for path in encrypted_files: + if "google-services" in path or "GoogleService" in path: + # We infer the destination from the file's directory, example: + # /scripts/gha-encrypted/auth/google-services.json.gpg turns into + # //auth/testapp/google-services.json + api = os.path.basename(os.path.dirname(path)) + if FLAGS.apis and api not in FLAGS.apis: + print("Skipping secret found in product api", api) + continue + print("Encrypted Google Service file found: %s" % path) + file_name = os.path.basename(path).replace(".gpg", "") + dest_paths = [os.path.join(repo_dir, api, "testapp", file_name)] + if FLAGS.artifact: + # ///auth/google-services.json + if "google-services" in path and os.path.isdir(os.path.join(repo_dir, FLAGS.artifact, api)): + dest_paths = [os.path.join(repo_dir, FLAGS.artifact, api, file_name)] + else: + continue + + decrypted_text = _decrypt(path, passphrase) + for dest_path in dest_paths: + with open(dest_path, "w") as f: + f.write(decrypted_text) + print("Copied decrypted google service file to %s" % dest_path) + # We use a Google Service file as the source of truth for the reverse id + # that needs to be patched into the Info.plist files. + if dest_path.endswith(".plist"): + _patch_reverse_id(dest_path) + _patch_bundle_id(dest_path) + + if FLAGS.artifact: + return + +def _find_encrypted_files(directory_to_search): + """Returns a list of full paths to all files encrypted with gpg.""" + encrypted_files = [] + for prefix, _, files in os.walk(directory_to_search): + for relative_path in files: + if relative_path.endswith(".gpg"): + encrypted_files.append(os.path.join(prefix, relative_path)) + return encrypted_files + + +def _decrypt(encrypted_file, passphrase): + """Returns the decrypted contents of the given .gpg file.""" + print("Decrypting %s" % encrypted_file) + # Note: if setting check=True, be sure to catch the error and not rethrow it + # or print a traceback, as the message will include the passphrase. + result = subprocess.run( + args=[ + "gpg", + "--passphrase", passphrase, + "--quiet", + "--batch", + "--yes", + "--decrypt", + encrypted_file], + check=False, + text=True, + capture_output=True) + if result.returncode: + # Remove any instances of the passphrase from error before logging it. + raise RuntimeError(result.stderr.replace(passphrase, "****")) + print("Decryption successful") + # rstrip to eliminate a linebreak that GPG may introduce. + return result.stdout.rstrip() + + +def _patch_reverse_id(service_plist_path): + """Patches the Info.plist file with the reverse id from the Service plist.""" + print("Attempting to patch reverse id in Info.plist") + with open(service_plist_path, "rb") as f: + service_plist = plistlib.load(f) + _patch_file( + path=os.path.join(os.path.dirname(service_plist_path), "testapp", "Info.plist"), + placeholder="REPLACE_WITH_REVERSED_CLIENT_ID", + value=service_plist["REVERSED_CLIENT_ID"]) + + +def _patch_bundle_id(service_plist_path): + """Patches the Info.plist file with the bundle id from the Service plist.""" + print("Attempting to patch bundle id in Info.plist") + with open(service_plist_path, "rb") as f: + service_plist = plistlib.load(f) + _patch_file( + path=os.path.join(os.path.dirname(service_plist_path), "testapp", "Info.plist"), + placeholder="$(PRODUCT_BUNDLE_IDENTIFIER)", + value=service_plist["BUNDLE_ID"]) + + +def _patch_file(path, placeholder, value): + """Patches instances of the placeholder with the given value.""" + # Note: value may be sensitive, so do not log. + with open(path, "r") as f_read: + text = f_read.read() + # Count number of times placeholder appears for debugging purposes. + replacements = text.count(placeholder) + patched_text = text.replace(placeholder, value) + with open(path, "w") as f_write: + f_write.write(patched_text) + print("Patched %d instances of %s in %s" % (replacements, placeholder, path)) + + +if __name__ == "__main__": + app.run(main)