diff --git a/.github/scripts/detect-app-cache.sh b/.github/scripts/detect-app-cache.sh new file mode 100644 index 0000000000000..293383c31cfb0 --- /dev/null +++ b/.github/scripts/detect-app-cache.sh @@ -0,0 +1,296 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2025 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Script to detect which apps need building vs. can be restored from cache +# Supports multiple cache sources: GitHub Actions cache and JFrog Artifactory +# Outputs JSON arrays for apps to build and apps to restore + +set -e # Exit on error +set -u # Exit on undefined variable +set -o pipefail # Exit if any command in pipeline fails + +# Required environment variables +: "${GH_TOKEN:?GH_TOKEN not set}" +: "${CACHE_VERSION:?CACHE_VERSION not set}" +: "${FORCE_REBUILD:?FORCE_REBUILD not set}" +: "${ARTIFACTORY_REPOSITORY_SNAPSHOT:?ARTIFACTORY_REPOSITORY_SNAPSHOT not set}" + +# Optional JFrog variables +JF_URL="${JF_URL:-}" +JF_USER="${JF_USER:-}" +JF_ACCESS_TOKEN="${JF_ACCESS_TOKEN:-}" + +# Input: MATRIX (JSON array of app configurations) +# Input: GITHUB_REF (current GitHub ref) +# Input: GITHUB_STEP_SUMMARY (path to step summary file) + +# Outputs to $GITHUB_OUTPUT: +# - apps_to_build: JSON array of apps that need building +# - apps_to_restore: JSON array of apps that can be restored from cache +# - apps_sha_map: JSON object mapping app names to their SHAs +# - has_apps_to_build: boolean flag +# - has_apps_to_restore: boolean flag + +echo "Collecting app SHAs and checking cache status..." +echo "Force rebuild mode: $FORCE_REBUILD" +echo "" + +# Setup JFrog CLI if credentials are available +JFROG_AVAILABLE="false" +echo "=== JFrog Setup ===" +echo "JF_URL present: $([ -n "$JF_URL" ] && echo 'YES' || echo 'NO')" +echo "JF_USER present: $([ -n "$JF_USER" ] && echo 'YES' || echo 'NO')" +echo "JF_ACCESS_TOKEN present: $([ -n "$JF_ACCESS_TOKEN" ] && echo 'YES' || echo 'NO')" + +if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "✓ All JFrog credentials available" + echo "Installing JFrog CLI..." + # Install JFrog CLI + curl -fL https://install-cli.jfrog.io | sh + export PATH=$PATH:$PWD + echo "JFrog CLI version: $(jf --version)" + + # Configure JFrog + echo "Configuring JFrog server: $JF_URL" + jf config add jfrog-server --url="$JF_URL" --user="$JF_USER" --access-token="$JF_ACCESS_TOKEN" --interactive=false + + # Test connection with verbose output + echo "Testing JFrog connection..." + if jf rt ping; then + JFROG_AVAILABLE="true" + echo "✓ JFrog connection successful" + echo "Repository: $ARTIFACTORY_REPOSITORY_SNAPSHOT" + else + echo "⚠ JFrog ping failed, will fall back to GitHub cache" + echo "Ping output was unsuccessful" + fi +else + echo "⚠ JFrog credentials not available, using GitHub cache only" + [ -z "$JF_URL" ] && echo " - Missing: JF_URL" + [ -z "$JF_USER" ] && echo " - Missing: JF_USER" + [ -z "$JF_ACCESS_TOKEN" ] && echo " - Missing: JF_ACCESS_TOKEN" +fi +echo "JFROG_AVAILABLE=$JFROG_AVAILABLE" +echo "===================" +echo "" + +# Get the matrix from input (passed as argument) +MATRIX="$1" + +# Build JSON arrays for apps that need building/restoring +APPS_TO_BUILD="[]" +APPS_TO_RESTORE="[]" +APPS_CHECKED=0 +APPS_CACHED=0 +APPS_IN_JFROG=0 +APPS_TO_BUILD_COUNT=0 +APPS_TO_RESTORE_COUNT=0 +APPS_SHA_MAP="{}" +echo "" + +echo "### 📦 Cache Status Report for ($GITHUB_REF)" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" +if [ "$FORCE_REBUILD" == "true" ]; then + echo "**🔄 FORCE REBUILD MODE ENABLED** - All caches bypassed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +if [ "$JFROG_AVAILABLE" == "true" ]; then + echo "**🎯 JFrog Artifact Cache**: Enabled for all branches" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +echo "| App | SHA | Cache Key | Status |" >> "$GITHUB_STEP_SUMMARY" +echo "|-----|-----|-----------|--------|" >> "$GITHUB_STEP_SUMMARY" + +# Iterate through each app in the matrix +while IFS= read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + + APPS_CHECKED=$((APPS_CHECKED + 1)) + + # Get current submodule SHA + if [ -d "$APP_PATH" ]; then + CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") + else + echo "⊘ $APP_NAME - directory not found, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Directory not found |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + if [ -z "$CURRENT_SHA" ]; then + echo "⊘ $APP_NAME - not a git repo, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Not a git repo |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Add SHA to the map for all apps (regardless of cache status) + APPS_SHA_MAP=$(echo "$APPS_SHA_MAP" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '.[$app] = $sha') + + # Cache key that would be used for this app + # Format: -app-build-- + CACHE_KEY="${CACHE_VERSION}-app-build-${APP_NAME}-${CURRENT_SHA}" + SHORT_SHA="${CURRENT_SHA:0:8}" + + echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " + + # If force rebuild is enabled, skip cache check and rebuild everything + if [ "$FORCE_REBUILD" == "true" ]; then + echo "🔄 force rebuild" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔄 Force rebuild |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check JFrog first before GitHub cache (available for all branches) + if [ "$JFROG_AVAILABLE" == "true" ]; then + JFROG_PATH="${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${CURRENT_SHA}.tar.gz" + + echo "" + echo " 🔍 Checking JFrog for $APP_NAME..." + echo " Path: $JFROG_PATH" + echo " Full SHA: $CURRENT_SHA" + + # Check if artifact exists in JFrog with verbose output + echo " Running: jf rt s \"$JFROG_PATH\"" + SEARCH_OUTPUT=$(jf rt s "$JFROG_PATH" 2>&1) + SEARCH_EXIT_CODE=$? + + echo " Search exit code: $SEARCH_EXIT_CODE" + if [ $SEARCH_EXIT_CODE -eq 0 ]; then + echo " Search output:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + + if echo "$SEARCH_OUTPUT" | grep -q "$JFROG_PATH"; then + echo " ✓ Artifact found in JFrog!" + echo "✓ in JFrog" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$JFROG_PATH\` | 📦 In JFrog |" >> "$GITHUB_STEP_SUMMARY" + APPS_IN_JFROG=$((APPS_IN_JFROG + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + # Add to restore list with JFrog source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg jfrog_path "$JFROG_PATH" --arg source "jfrog" '. + [($app + {sha: $sha, jfrog_path: $jfrog_path, source: $source})]') + continue + else + echo " ✗ Artifact not found in search results" + fi + else + echo " ✗ Search failed with error:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + fi + echo " → Falling back to GitHub cache check" + fi + + # Check if cache exists using GitHub CLI + # Include --ref to access caches from the current ref (branch, PR, etc.) + CACHE_EXISTS="false" + if ! CACHE_LIST=$(gh cache list --ref "$GITHUB_REF" --key "$CACHE_KEY" --json key --jq ".[].key" 2>&1); then + echo "⚠️ Warning: Failed to query cache for $APP_NAME: $CACHE_LIST" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⚠️ Cache check failed - will build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + if echo "$CACHE_LIST" | grep -q "^${CACHE_KEY}$"; then + CACHE_EXISTS="true" + APPS_CACHED=$((APPS_CACHED + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + echo "✓ cached" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> "$GITHUB_STEP_SUMMARY" + # Add to restore list with GitHub cache source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg cache_key "$CACHE_KEY" --arg source "github-cache" '. + [($app + {sha: $sha, cache_key: $cache_key, source: $source})]') + else + echo "⚡ needs build" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + fi + +done < <(echo "$MATRIX" | jq -c '.[]') + +echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY" +echo "- Total apps checked: $APPS_CHECKED" >> "$GITHUB_STEP_SUMMARY" +echo "- 📦 Apps in JFrog: $APPS_IN_JFROG" >> "$GITHUB_STEP_SUMMARY" +echo "- ✅ Apps with cached builds: $APPS_CACHED" >> "$GITHUB_STEP_SUMMARY" +echo "- 🔨 Apps needing build: $APPS_TO_BUILD_COUNT" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" + +TOTAL_AVAILABLE=$((APPS_IN_JFROG + APPS_CACHED)) +if [ $TOTAL_AVAILABLE -gt 0 ] && [ $APPS_CHECKED -gt 0 ]; then + CACHE_HIT_PERCENT=$((TOTAL_AVAILABLE * 100 / APPS_CHECKED)) + echo "**Cache hit rate: ${CACHE_HIT_PERCENT}%** 🎯" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi + +echo "" +echo "Summary:" +echo " Total apps: $APPS_CHECKED" +echo " In JFrog: $APPS_IN_JFROG" +echo " Cached: $APPS_CACHED" +echo " To build: $APPS_TO_BUILD_COUNT" + +# Validate no duplicate apps in build and restore lists +BUILD_APPS=$(echo "$APPS_TO_BUILD" | jq -r '.[].name' | sort) +RESTORE_APPS=$(echo "$APPS_TO_RESTORE" | jq -r '.[].name' | sort) +DUPLICATE_APPS=$(comm -12 <(echo "$BUILD_APPS") <(echo "$RESTORE_APPS")) + +if [ -n "$DUPLICATE_APPS" ]; then + echo "ERROR: Apps appear in both build and restore lists:" + echo "$DUPLICATE_APPS" + exit 1 +fi + +# Validate that we built valid JSON +if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_build" + echo "Content: $APPS_TO_BUILD" + exit 1 +fi + +if ! echo "$APPS_TO_RESTORE" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_restore" + echo "Content: $APPS_TO_RESTORE" + exit 1 +fi + +# Output app list with SHAs for the build job to use +# Use proper multiline output format for GitHub Actions +echo "apps_to_build<> "$GITHUB_OUTPUT" +echo "$APPS_TO_BUILD" >> "$GITHUB_OUTPUT" +echo "APPS_TO_BUILD_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the unified list of apps to restore (from either GitHub cache or JFrog) +echo "apps_to_restore<> "$GITHUB_OUTPUT" +echo "$APPS_TO_RESTORE" >> "$GITHUB_OUTPUT" +echo "APPS_TO_RESTORE_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the SHA map for all apps +echo "apps_sha_map<> "$GITHUB_OUTPUT" +echo "$APPS_SHA_MAP" >> "$GITHUB_OUTPUT" +echo "APPS_SHA_MAP_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output flags for conditional job execution +if [ $APPS_TO_BUILD_COUNT -gt 0 ]; then + echo "has_apps_to_build=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_build=false" >> "$GITHUB_OUTPUT" +fi + +if [ $APPS_TO_RESTORE_COUNT -gt 0 ]; then + echo "has_apps_to_restore=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_restore=false" >> "$GITHUB_OUTPUT" +fi + +echo "" +if [ $APPS_TO_BUILD_COUNT -eq 0 ]; then + echo "🎉 All apps are cached! No builds needed." +else + echo "✓ Will build $APPS_TO_BUILD_COUNT app(s)" +fi diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index a2cbfdb849649..b167639ada379 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -1,12 +1,13 @@ -name: Build Nextcloud Workspace artifact +name: Build Nextcloud Workspace artifact (Optimized) # SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors # SPDX-FileCopyrightText: 2025 STRATO AG # SPDX-License-Identifier: AGPL-3.0-or-later -# The Nextcloud Workspace source is packaged as a container image. -# This is a workaround because releases cannot be created without tags, -# and we want to be able to create snapshots from branches. +# Optimized build workflow using GitHub Actions cache +# - Checks GitHub Actions cache for each app's current SHA +# - Only builds apps without cached artifacts +# - Significantly reduces build time by reusing cached builds via GitHub Actions cache on: pull_request: @@ -30,6 +31,13 @@ on: branches: - ionos-dev - ionos-stable + workflow_dispatch: # Manual trigger to bypass all caches + inputs: + force_rebuild: + description: 'Force rebuild all apps and dependencies (bypass ALL caches)' + required: false + type: boolean + default: false concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/ionos-dev' && github.run_id || github.event.pull_request.number || github.ref }} @@ -40,6 +48,10 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-ncwserver-snapshot + # Cache version - increment this to invalidate all caches when build process changes + # Update when: Node.js version changes, PHP version changes, build scripts modified, etc. + # Format: v. (e.g., v1.0, v1.1, v2.0) + CACHE_VERSION: v1.0 permissions: contents: read @@ -48,473 +60,104 @@ jobs: prepare-matrix: runs-on: ubuntu-latest outputs: - external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} + apps_to_build: ${{ steps.detect.outputs.apps_to_build }} + apps_to_restore: ${{ steps.detect.outputs.apps_to_restore }} + external_apps_matrix: ${{ steps.set_matrix.outputs.matrix }} + apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }} + has_apps_to_build: ${{ steps.detect.outputs.has_apps_to_build }} + has_apps_to_restore: ${{ steps.detect.outputs.has_apps_to_restore }} steps: - name: Checkout repository uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 # Shallow clone - only need current submodule SHAs for cache detection - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y make jq - - name: Set matrix - id: set-matrix + - name: Check configuration run: | - # Create matrix configuration as a compact JSON string - matrix='[ - { - "name": "activity", - "path": "apps-external/activity", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_activity_app" - }, - { - "name": "assistant", - "path": "apps-external/assistant", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_assistant_app" - }, - { - "name": "calendar", - "path": "apps-external/calendar", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_calendar_app" - }, - { - "name": "circles", - "path": "apps-external/circles", - "has_npm": false, - "has_composer": true, - "makefile_target": "build_circles_app" - }, - { - "name": "collectives", - "path": "apps-external/collectives", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_collectives_app" - }, - { - "name": "contacts", - "path": "apps-external/contacts", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_contacts_app" - }, - { - "name": "deck", - "path": "apps-external/deck", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_deck_app" - }, - { - "name": "end_to_end_encryption", - "path": "apps-external/end_to_end_encryption", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_end_to_end_encryption_app" - }, - { - "name": "forms", - "path": "apps-external/forms", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_forms_app" - }, - { - "name": "groupfolders", - "path": "apps-external/groupfolders", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_groupfolders_app" - }, - { - "name": "integration_openai", - "path": "apps-external/integration_openai", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_integration_openai_app" - }, - { - "name": "mail", - "path": "apps-external/mail", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_mail_app" - }, - { - "name": "ncw_apps_menu", - "path": "apps-external/ncw_apps_menu", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_ncw_apps_menu_app" - }, - { - "name": "ncw_mailtemplate", - "path": "apps-external/ncw_mailtemplate", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_ncw_mailtemplate_app" - }, - { - "name": "notes", - "path": "apps-external/notes", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_notes_app" - }, - { - "name": "notifications", - "path": "apps-external/notifications", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_notifications_app" - }, - { - "name": "notify_push", - "path": "apps-external/notify_push", - "has_npm": false, - "has_composer": true, - "makefile_target": "build_notify_push_app" - }, - { - "name": "password_policy", - "path": "apps-external/password_policy", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_password_policy_app" - }, - { - "name": "richdocuments", - "path": "apps-external/richdocuments", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_richdocuments_app" - }, - { - "name": "spreed", - "path": "apps-external/spreed", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_spreed_app" - }, - { - "name": "tables", - "path": "apps-external/tables", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_tables_app" - }, - { - "name": "tasks", - "path": "apps-external/tasks", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_tasks_app" - }, - { - "name": "text", - "path": "apps-external/text", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_text_app" - }, - { - "name": "twofactor_totp", - "path": "apps-external/twofactor_totp", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_twofactor_totp_app" - }, - { - "name": "user_oidc", - "path": "apps-external/user_oidc", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_user_oidc_app" - }, - { - "name": "viewer", - "path": "apps-external/viewer", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_viewer_app" - }, - { - "name": "whiteboard", - "path": "apps-external/whiteboard", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_whiteboard_app" - } - ]' - - # Validate JSON and output as compact format - if echo "$matrix" | jq empty 2>/dev/null; then - echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT - echo "Matrix configuration set successfully" - else - echo "Error: Invalid JSON in matrix configuration" - exit 1 - fi - - - name: Validate matrix against Makefile - run: | - set +e # Intentionally allow script to continue on error for custom error handling and reporting to GITHUB_STEP_SUMMARY - set -u # Exit on undefined variable - - echo "### 🔍 Matrix Validation" >> $GITHUB_STEP_SUMMARY + echo "" + echo "### 🔧 Remote Trigger Configuration" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**DISABLE_REMOTE_TRIGGER value:** \`${{ vars.DISABLE_REMOTE_TRIGGER }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - # Debug: Check if apps-external exists - echo "Checking apps-external directory..." - if [ ! -d "apps-external" ]; then - echo "❌ **Error:** apps-external directory does not exist!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Directory listing:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - ls -la >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - echo "Apps-external directory exists. Listing contents:" - ls -la apps-external/ | head -10 + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "🔧 Remote Trigger Configuration" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "DISABLE_REMOTE_TRIGGER = '${{ vars.DISABLE_REMOTE_TRIGGER }}'" + echo "" - # Check if jq is available - echo "Checking if jq is installed..." - if ! command -v jq &> /dev/null; then - echo "❌ **Error:** jq is not installed!" >> $GITHUB_STEP_SUMMARY + if [ "${{ vars.DISABLE_REMOTE_TRIGGER }}" == "true" ]; then + echo "⚠️ Remote trigger is DISABLED" + echo " The 'trigger-remote-dev-workflow' job will be SKIPPED" + echo "**Status:** ⚠️ Remote trigger is **DISABLED**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "jq is required for matrix generation but was not found in PATH." >> $GITHUB_STEP_SUMMARY - exit 1 - fi - echo "jq version: $(jq --version)" - - echo "Generating matrix from Makefile..." - # Capture both stdout and stderr separately to better diagnose issues - makefile_output=$(make -f IONOS/Makefile generate_external_apps_matrix_json 2>&1) - makefile_exit_code=$? - - echo "Makefile exit code: ${makefile_exit_code}" - echo "Makefile output length: ${#makefile_output}" - - # Debug: Check if GITHUB_STEP_SUMMARY is set - echo "GITHUB_STEP_SUMMARY: ${GITHUB_STEP_SUMMARY:-NOT SET}" - - # If the Makefile command failed, show the error - if [ ${makefile_exit_code} -ne 0 ]; then - echo "" - echo "=== MAKEFILE ERROR ===" - echo "Exit code: ${makefile_exit_code}" - echo "Output:" - echo "$makefile_output" - echo "=====================" - echo "" - - # Write to summary - echo "❌ **Error:** Makefile command failed with exit code ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY + echo "The \`trigger-remote-dev-workflow\` job will be skipped." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Makefile error output" >> $GITHUB_STEP_SUMMARY + echo "To enable, delete the variable or set it to a value other than 'true' at:" >> $GITHUB_STEP_SUMMARY + echo "https://github.com/${{ github.repository }}/settings/variables/actions" >> $GITHUB_STEP_SUMMARY + else + echo "✅ Remote trigger is ENABLED" + echo " The 'trigger-remote-dev-workflow' job will proceed" + echo "**Status:** ✅ Remote trigger is **ENABLED**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$makefile_output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - echo "Error written to summary file: ${GITHUB_STEP_SUMMARY}" - exit 1 + echo "The \`trigger-remote-dev-workflow\` job will proceed to trigger the remote GitLab workflow." >> $GITHUB_STEP_SUMMARY fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - # Filter out the info message to get just the JSON - # The Makefile outputs "[i] Generating..." to stderr, but we captured everything with 2>&1 - # So we need to extract just the JSON part - generated_matrix=$(echo "$makefile_output" | grep -v '^\[i\]' || echo "$makefile_output") - - workflow_matrix='${{ steps.set-matrix.outputs.matrix }}' - - # Debug output - echo "Generated matrix length: ${#generated_matrix}" - echo "Workflow matrix length: ${#workflow_matrix}" + - name: List caches before restore + run: gh cache list + env: + GH_TOKEN: ${{ github.token }} - # Show first 200 chars of generated matrix for debugging - if [ -n "$generated_matrix" ]; then - echo "Generated matrix preview: ${generated_matrix:0:200}..." - fi + - name: Generate apps matrix dynamically from Makefile + id: set_matrix + run: | + # Generate matrix from Makefile - single source of truth + echo "Generating matrix from Makefile..." + matrix_output=$(make -f IONOS/Makefile generate_external_apps_matrix_json 2>&1) - # Validate that we got valid JSON - if ! echo "$generated_matrix" | jq empty 2>/dev/null; then - echo "❌ **Error:** Generated matrix is not valid JSON" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Invalid JSON output" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - exit 1 + # Filter out info messages to get just the JSON + # Note: Use same filtering logic as validation script to ensure consistency + if echo "$matrix_output" | grep -q '^\[i\]'; then + matrix=$(echo "$matrix_output" | grep -v '^\[i\]') + else + matrix="$matrix_output" fi - # Validate that we got data - if [ -z "$generated_matrix" ] || [ -z "$workflow_matrix" ]; then - echo "❌ **Error:** Failed to load matrices" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- Generated matrix empty: $([ -z "$generated_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY - echo "- Workflow matrix empty: $([ -z "$workflow_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY - echo "- Makefile exit code: ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY - + # Validate JSON + if ! echo "$matrix" | jq empty 2>/dev/null; then + echo "Error: Generated matrix is not valid JSON" + echo "Output: $matrix_output" exit 1 fi - # Sort both matrices for comparison - generated_sorted=$(echo "$generated_matrix" | jq -S '.' 2>&1 || echo "ERROR") - workflow_sorted=$(echo "$workflow_matrix" | jq -S '.' 2>&1 || echo "ERROR") + # Output as compact format + echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT + echo "Matrix generated successfully with $(echo "$matrix" | jq 'length') apps" - echo "Sorted matrix lengths - generated: ${#generated_sorted}, workflow: ${#workflow_sorted}" - - # Compare the two matrices - if [ "$generated_sorted" = "$workflow_sorted" ]; then - echo "✅ **Validation passed!** The workflow matrix matches the Makefile configuration." >> $GITHUB_STEP_SUMMARY - echo "" - echo "✅ Matrix validation passed!" - else - echo "❌ **Validation failed!** The workflow matrix does not match the Makefile configuration." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - echo "Starting detailed comparison..." - - # Extract app names from both matrices - generated_apps=$(echo "$generated_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") - workflow_apps=$(echo "$workflow_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") - - echo "Generated apps count: $(echo "$generated_apps" | wc -l)" - echo "Workflow apps count: $(echo "$workflow_apps" | wc -l)" - - # Find missing apps (in Makefile but not in workflow) - missing_apps=$(comm -23 <(echo "$generated_apps") <(echo "$workflow_apps")) - if [ $? -ne 0 ]; then - echo "Error: comm command failed when finding missing apps." >&2 - exit 1 - fi - # Find extra apps (in workflow but not in Makefile) - extra_apps=$(comm -13 <(echo "$generated_apps") <(echo "$workflow_apps")) - if [ $? -ne 0 ]; then - echo "Error: comm command failed when finding extra apps." >&2 - exit 1 - fi - - echo "Missing apps: ${missing_apps:-none}" - echo "Extra apps: ${extra_apps:-none}" - - if [ -n "$missing_apps" ]; then - echo "#### ⚠️ Missing Apps" >> $GITHUB_STEP_SUMMARY - echo "The following apps are configured in the Makefile but missing from the workflow:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$missing_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - - if [ -n "$extra_apps" ]; then - echo "#### ⚠️ Extra Apps" >> $GITHUB_STEP_SUMMARY - echo "The following apps are in the workflow but not configured in the Makefile:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$extra_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - - # Check for configuration mismatches in common apps - common_apps=$(comm -12 <(echo "$generated_apps") <(echo "$workflow_apps") 2>/dev/null || echo "") - - echo "Common apps count: $(echo "$common_apps" | wc -l)" - - if [ -n "$common_apps" ]; then - mismatched_apps="" - - while IFS= read -r app; do - [ -z "$app" ] && continue - gen_config=$(echo "$generated_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") - wf_config=$(echo "$workflow_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") - - if [ -n "$gen_config" ] && [ -n "$wf_config" ] && [ "$gen_config" != "$wf_config" ]; then - mismatched_apps="${mismatched_apps}${app}"$'\n' - fi - done <<< "$common_apps" - - echo "Mismatched apps: ${mismatched_apps:-none}" - - if [ -n "$mismatched_apps" ]; then - echo "#### ⚠️ Configuration Mismatches" >> $GITHUB_STEP_SUMMARY - echo "The following apps have different configurations (has_npm, has_composer, etc.):" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$mismatched_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "📋 Detailed differences" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - - while IFS= read -r app; do - [ -z "$app" ] && continue - echo "=== $app ===" >> $GITHUB_STEP_SUMMARY - diff -u --label "Workflow" --label "Makefile" \ - <(echo "$workflow_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ - <(echo "$generated_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ - >> $GITHUB_STEP_SUMMARY 2>&1 || true - done <<< "$mismatched_apps" - - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - fi - - # Provide fix instructions - echo "#### 🔧 How to Fix" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Run this command locally to generate the correct matrix:" >> $GITHUB_STEP_SUMMARY - echo '```bash' >> $GITHUB_STEP_SUMMARY - echo "make -f IONOS/Makefile generate_external_apps_matrix_json" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Then update the \`matrix\` variable in \`.github/workflows/build-artifact.yml\` in the set-matrix step with the generated output." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Show full diff in expandable section - echo "
" >> $GITHUB_STEP_SUMMARY - echo "📄 Full matrix comparison" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Workflow Matrix:**" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - echo "$workflow_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$workflow_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Makefile Matrix:**" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - echo "$generated_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - echo "" - echo "❌ ERROR: Matrix validation failed!" - echo "See the job summary for details on what's wrong and how to fix it." - echo "Summary file size: $(wc -c < $GITHUB_STEP_SUMMARY || echo 0) bytes" - exit 1 - fi + - name: Collect apps and their SHAs for cache-based building + id: detect + env: + GH_TOKEN: ${{ github.token }} + CACHE_VERSION: ${{ env.CACHE_VERSION }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }} + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }} + GITHUB_REF: ${{ github.ref }} + run: | + bash .github/scripts/detect-app-cache.sh '${{ steps.set_matrix.outputs.matrix }}' build-external-apps: runs-on: ubuntu-latest needs: prepare-matrix + # Only run if there are actually apps to build (prevents runner from starting) + if: needs.prepare-matrix.outputs.has_apps_to_build == 'true' permissions: contents: read @@ -523,56 +166,188 @@ jobs: strategy: max-parallel: 20 matrix: - app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + # Use the filtered list of apps that need building (not in cache) + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_build) }} steps: + - name: Get app configuration from full matrix + id: app-config + run: | + # Get the full matrix to look up app configuration + FULL_MATRIX='${{ needs.prepare-matrix.outputs.external_apps_matrix }}' + APP_NAME='${{ matrix.app_info.name }}' + + # Find the app configuration in the full matrix + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') + + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find configuration for $APP_NAME" + exit 1 + fi + + # Extract configuration values + APP_PATH=$(echo "$APP_CONFIG" | jq -r '.path') + HAS_NPM=$(echo "$APP_CONFIG" | jq -r '.has_npm') + HAS_COMPOSER=$(echo "$APP_CONFIG" | jq -r '.has_composer') + MAKEFILE_TARGET=$(echo "$APP_CONFIG" | jq -r '.makefile_target') + + # Set outputs + echo "path=$APP_PATH" >> $GITHUB_OUTPUT + echo "has-npm=$HAS_NPM" >> $GITHUB_OUTPUT + echo "has-composer=$HAS_COMPOSER" >> $GITHUB_OUTPUT + echo "makefile-target=$MAKEFILE_TARGET" >> $GITHUB_OUTPUT + echo "Building $APP_NAME from $APP_PATH (SHA: ${{ matrix.app_info.sha }})" + - name: Checkout server uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 - name: Set up node with version from package.json's engines - if: matrix.app.has_npm - uses: actions/setup-node@v5 + if: steps.app-config.outputs.has-npm == 'true' + uses: actions/setup-node@v6 with: node-version-file: "package.json" cache: 'npm' - cache-dependency-path: ${{ matrix.app.path }}/package-lock.json + cache-dependency-path: ${{ steps.app-config.outputs.path }}/package-lock.json - name: Setup PHP with PECL extension - if: matrix.app.has_composer + if: steps.app-config.outputs.has-composer == 'true' uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: + php-version: '8.3' tools: composer:v2 extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache env: runner: ubuntu-latest - - name: Cache Composer dependencies for ${{ matrix.app.name }} - if: matrix.app.has_composer + - name: Cache Composer dependencies for ${{ matrix.app_info.name }} + if: steps.app-config.outputs.has-composer == 'true' && github.event.inputs.force_rebuild != 'true' uses: actions/cache@v4 with: - path: ${{ matrix.app.path }}/vendor - key: ${{ runner.os }}-composer-${{ matrix.app.name }}-${{ hashFiles(format('{0}/composer.lock', matrix.app.path)) }} + path: ${{ steps.app-config.outputs.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }} restore-keys: | - ${{ runner.os }}-composer-${{ matrix.app.name }}- + ${{ runner.os }}-composer-${{ matrix.app_info.name }}- - - name: Build ${{ matrix.app.name }} app - run: make -f IONOS/Makefile ${{ matrix.app.makefile_target }} + - name: Build ${{ matrix.app_info.name }} app + run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} - - name: Upload ${{ matrix.app.name }} build artifacts + - name: Report build completion + if: success() + run: | + echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ matrix.app_info.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Save built app to cache for future runs + - name: Save build artifacts to cache + uses: actions/cache/save@v4 + with: + path: ${{ steps.app-config.outputs.path }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} + + # Push to JFrog for ionos-dev branch builds + - name: Setup JFrog CLI + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Push ${{ matrix.app_info.name }} to JFrog + run: | + set -e + APP_NAME="${{ matrix.app_info.name }}" + APP_SHA="${{ matrix.app_info.sha }}" + APP_PATH="${{ steps.app-config.outputs.path }}" + + echo "=== JFrog Upload Debug Info ===" + echo "📦 Packaging $APP_NAME for JFrog upload..." + echo "App Name: $APP_NAME" + echo "App SHA: $APP_SHA" + echo "App Path: $APP_PATH" + echo "Repository: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}" + echo "===============================" + + # Verify app path exists + if [ ! -d "$APP_PATH" ]; then + echo "❌ ERROR: App path does not exist: $APP_PATH" + exit 1 + fi + + echo "App directory contents (top level):" + ls -la "$APP_PATH" | head -20 + + # Create tar.gz archive of the built app (excluding node_modules and other build artifacts) + ARCHIVE_NAME="${APP_NAME}-${APP_SHA}.tar.gz" + echo "" + echo "Creating archive: $ARCHIVE_NAME" + echo "Running: tar -czf \"$ARCHIVE_NAME\" --exclude=\"node_modules\" --exclude=\".git\" --exclude=\"*.log\" -C \"$(dirname "$APP_PATH")\" \"$(basename "$APP_PATH")\"" + + tar -czf "$ARCHIVE_NAME" \ + --exclude="node_modules" \ + --exclude=".git" \ + --exclude="*.log" \ + -C "$(dirname "$APP_PATH")" \ + "$(basename "$APP_PATH")" + + echo "✓ Archive created successfully" + echo "Archive size:" + ls -lh "$ARCHIVE_NAME" + + # Upload to JFrog - store in snapshot repo under dev/apps/ + # Include CACHE_VERSION in path to enable complete cache invalidation + JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${{ env.CACHE_VERSION }}/${APP_NAME}/${ARCHIVE_NAME}" + + echo "" + echo "Uploading to JFrog..." + echo "Target Path: $JFROG_PATH" + echo "Properties: app.name=${APP_NAME};app.sha=${APP_SHA};vcs.branch=${{ github.ref_name }};vcs.revision=${{ github.sha }}" + echo "Running: jf rt upload \"$ARCHIVE_NAME\" \"$JFROG_PATH\" --target-props \"...\"" + + if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" \ + --target-props "app.name=${APP_NAME};app.sha=${APP_SHA};vcs.branch=${{ github.ref_name }};vcs.revision=${{ github.sha }}"; then + echo "✅ Successfully uploaded $APP_NAME to JFrog" + echo "" + echo "Verifying upload..." + if jf rt s "$JFROG_PATH"; then + echo "✓ Upload verified - artifact is accessible in JFrog" + else + echo "⚠ Upload succeeded but verification search failed" + fi + else + UPLOAD_EXIT_CODE=$? + echo "❌ Failed to upload to JFrog (exit code: $UPLOAD_EXIT_CODE)" + echo "⚠️ Continuing workflow despite upload failure..." + fi + + # Clean up archive + echo "" + echo "Cleaning up local archive..." + rm -f "$ARCHIVE_NAME" + echo "✓ Cleanup complete" + + - name: Upload ${{ matrix.app_info.name }} build artifacts uses: actions/upload-artifact@v4 with: retention-days: 1 - name: external-app-build-${{ matrix.app.name }} + name: external-app-build-${{ matrix.app_info.name }} path: | - ${{ matrix.app.path }} - !${{ matrix.app.path }}/node_modules + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules build-artifact: runs-on: ubuntu-latest needs: [prepare-matrix, build-external-apps] + # Always run this job, even if build-external-apps is skipped + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') permissions: contents: read @@ -586,7 +361,92 @@ jobs: uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 + + - name: Setup JFrog CLI + if: needs.prepare-matrix.outputs.has_apps_to_restore == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Restore cached apps + if: needs.prepare-matrix.outputs.has_apps_to_restore == 'true' + run: | + set -e + + echo "📦 Restoring cached apps..." + APPS_TO_RESTORE='${{ needs.prepare-matrix.outputs.apps_to_restore }}' + + # Process each app in the restore list + echo "$APPS_TO_RESTORE" | jq -c '.[]' | while read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_SHA=$(echo "$app_json" | jq -r '.sha') + APP_PATH=$(echo "$app_json" | jq -r '.path') + SOURCE=$(echo "$app_json" | jq -r '.source') + + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "Restoring: $APP_NAME (source: $SOURCE)" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if [ "$SOURCE" == "jfrog" ]; then + # Restore from JFrog + JFROG_PATH=$(echo "$app_json" | jq -r '.jfrog_path') + ARCHIVE_NAME="${APP_NAME}-${APP_SHA}.tar.gz" + + echo "📥 Downloading from JFrog: $JFROG_PATH" + + if jf rt download "$JFROG_PATH" "$ARCHIVE_NAME" --flat=true; then + echo "✅ Downloaded successfully" + echo "Extracting to $APP_PATH..." + mkdir -p "$(dirname "$APP_PATH")" + tar -xzf "$ARCHIVE_NAME" -C "$(dirname "$APP_PATH")" + + if [ -d "$APP_PATH" ] && [ -f "$APP_PATH/appinfo/info.xml" ]; then + echo "✅ Restored $APP_NAME from JFrog" + else + echo "❌ Failed to extract or validate $APP_NAME" + exit 1 + fi + + rm -f "$ARCHIVE_NAME" + else + echo "❌ Failed to download from JFrog" + exit 1 + fi + + elif [ "$SOURCE" == "github-cache" ]; then + # Restore from GitHub cache + CACHE_KEY=$(echo "$app_json" | jq -r '.cache_key') + + echo "💾 Restoring from GitHub cache: $CACHE_KEY" + + # Use actions/cache/restore in a way that works in a script context + # We need to use gh CLI to restore the cache + if gh cache restore "$CACHE_KEY" --key "$CACHE_KEY"; then + echo "✅ Restored $APP_NAME from GitHub cache" + + # Validate restoration + if [ ! -d "$APP_PATH" ] || [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Validation failed for $APP_NAME" + exit 1 + fi + else + echo "❌ Failed to restore from GitHub cache" + exit 1 + fi + else + echo "❌ Unknown source: $SOURCE" + exit 1 + fi + done + + echo "" + echo "✅ All cached apps restored successfully" + env: + GH_TOKEN: ${{ github.token }} - name: Download build external apps uses: actions/download-artifact@v5 @@ -639,10 +499,10 @@ jobs: done - name: Set up node with version from package.json's engines - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version-file: "package.json" - cache: 'npm' + cache: ${{ github.event.inputs.force_rebuild != 'true' && 'npm' || '' }} - name: Install Dependencies run: sudo apt-get update && sudo apt-get install -y make zip unzip @@ -653,12 +513,14 @@ jobs: - name: Setup PHP with PECL extension uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: + php-version: '8.3' tools: composer:v2 extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache env: runner: ubuntu-latest - name: Cache Composer dependencies + if: github.event.inputs.force_rebuild != 'true' uses: actions/cache@v4 with: path: vendor @@ -709,7 +571,12 @@ jobs: upload-to-artifactory: runs-on: self-hosted # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches - if: github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' + if: | + always() && + (github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + needs.build-artifact.result == 'success' name: Push to artifactory needs: [prepare-matrix, build-external-apps, build-artifact] @@ -788,12 +655,39 @@ jobs: export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" - # Promote current build to the "latest" dev build - jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ - --build-name "${{ env.BUILD_NAME }}" \ - --build-number ${{ github.run_number }} \ - --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ - $PATH_TO_LATEST_ARTIFACT + # Upload with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + UPLOAD_SUCCESS=false + DELAY_SEC=10 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Upload attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ + $PATH_TO_LATEST_ARTIFACT; then + UPLOAD_SUCCESS=true + echo "✅ Upload successful on attempt $ATTEMPT" + break + else + echo "⚠️ Upload attempt $ATTEMPT failed" + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting $DELAY_SEC seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) # Exponential backoff: delays are 10s, then 20s (sleep occurs after failed attempts) + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$UPLOAD_SUCCESS" != "true" ]; then + echo "❌ Upload failed after $MAX_ATTEMPTS attempts" + exit 1 + fi echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT @@ -806,6 +700,12 @@ jobs: nextcloud-workspace-artifact-to-ghcr_io: runs-on: ubuntu-latest + # Only run if build-artifact succeeded + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + needs.build-artifact.result == 'success' permissions: contents: read @@ -864,13 +764,21 @@ jobs: git diff exit 1 # make it red to grab attention - trigger-remote-dev-worflow: + trigger-remote-dev-workflow: runs-on: self-hosted name: Trigger remote workflow - needs: [ build-artifact, upload-to-artifactory ] + needs: [prepare-matrix, build-artifact, upload-to-artifactory] # Trigger remote build on "ionos-dev|ionos-stable" branch *push* defined in the on:push:branches - if: github.event_name == 'push' && ( github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' ) + # Can be disabled via repository variable 'DISABLE_REMOTE_TRIGGER' (set to 'true' to disable) + # Configure at: https://github.com/IONOS-Productivity/ncw-server/settings/variables/actions + if: | + github.event_name == 'push' && + (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.prepare-matrix.result == 'success' && + needs.build-artifact.result == 'success' && + needs.upload-to-artifactory.result == 'success' && + vars.DISABLE_REMOTE_TRIGGER != 'true' steps: - name: Trigger remote workflow run: | @@ -887,22 +795,55 @@ jobs: BUILD_TYPE="stable" fi - # Trigger GitLab pipeline via webhook with build artifacts and metadata - # Passes GitHub context variables to remote GitLab workflow - curl \ - --silent \ - --insecure \ - --request POST \ - --fail-with-body \ - -o response.json \ - --form token=${{ secrets.GITLAB_TOKEN }} \ - --form ref="stable" \ - --form "variables[GITHUB_SHA]=${{ github.sha }}" \ - --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ - --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ - --form "variables[BUILD_ID]=${{ github.run_id }}" \ - --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ - "${{ secrets.GITLAB_TRIGGER_URL }}" || ( RETCODE="$?"; jq . response.json; exit "$RETCODE" ) + # Trigger GitLab pipeline via webhook with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + TRIGGER_SUCCESS=false + DELAY_SEC=5 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Trigger attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if curl \ + --silent \ + --insecure \ + --request POST \ + --fail-with-body \ + -o response.json \ + --form token=${{ secrets.GITLAB_TOKEN }} \ + --form ref="stable" \ + --form "variables[GITHUB_SHA]=${{ github.sha }}" \ + --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ + --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ + --form "variables[BUILD_ID]=${{ github.run_id }}" \ + --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + "${{ secrets.GITLAB_TRIGGER_URL }}"; then + TRIGGER_SUCCESS=true + echo "✅ Trigger successful on attempt $ATTEMPT" + break + else + RETCODE="$?" + echo "⚠️ Trigger attempt $ATTEMPT failed with code $RETCODE" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting ${DELAY_SEC} seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) # Exponential backoff: 5s, 10s, 20s + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$TRIGGER_SUCCESS" != "true" ]; then + echo "❌ Trigger failed after $MAX_ATTEMPTS attempts" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + exit 1 + fi # Disable command echo set +x