diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index a2cbfdb849649..a110ac203da71 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -1,12 +1,13 @@ -name: Build Nextcloud Workspace artifact +name: Build Nextcloud Workspace artifact (Optimized) # SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors # SPDX-FileCopyrightText: 2025 STRATO AG # SPDX-License-Identifier: AGPL-3.0-or-later -# The Nextcloud Workspace source is packaged as a container image. -# This is a workaround because releases cannot be created without tags, -# and we want to be able to create snapshots from branches. +# Optimized build workflow using GitHub Actions cache +# - Checks GitHub Actions cache for each app's current SHA +# - Only builds apps without cached artifacts +# - Significantly reduces build time by reusing cached builds via GitHub Actions cache on: pull_request: @@ -30,6 +31,13 @@ on: branches: - ionos-dev - ionos-stable + workflow_dispatch: # Manual trigger only for comparison testing + inputs: + force_rebuild: + description: 'Force rebuild all apps (ignore cache)' + required: false + type: boolean + default: false concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/ionos-dev' && github.run_id || github.event.pull_request.number || github.ref }} @@ -40,6 +48,10 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-ncwserver-snapshot + # Cache version - increment this to invalidate all caches when build process changes + # Update when: Node.js version changes, PHP version changes, build scripts modified, etc. + # Format: v. (e.g., v1.0, v1.1, v2.0) + CACHE_VERSION: v1.0 permissions: contents: read @@ -48,473 +60,206 @@ jobs: prepare-matrix: runs-on: ubuntu-latest outputs: - external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} + apps_to_build: ${{ steps.detect.outputs.apps_to_build }} + apps_to_restore: ${{ steps.detect.outputs.apps_to_restore }} + external_apps_matrix: ${{ steps.set_matrix.outputs.matrix }} + has_cached_apps: ${{ steps.detect.outputs.has_cached_apps }} + apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }} steps: - name: Checkout repository uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 # Shallow clone - only need current submodule SHAs for cache detection - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y make jq - - name: Set matrix - id: set-matrix + - name: Generate apps matrix dynamically from Makefile + id: set_matrix run: | - # Create matrix configuration as a compact JSON string - matrix='[ - { - "name": "activity", - "path": "apps-external/activity", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_activity_app" - }, - { - "name": "assistant", - "path": "apps-external/assistant", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_assistant_app" - }, - { - "name": "calendar", - "path": "apps-external/calendar", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_calendar_app" - }, - { - "name": "circles", - "path": "apps-external/circles", - "has_npm": false, - "has_composer": true, - "makefile_target": "build_circles_app" - }, - { - "name": "collectives", - "path": "apps-external/collectives", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_collectives_app" - }, - { - "name": "contacts", - "path": "apps-external/contacts", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_contacts_app" - }, - { - "name": "deck", - "path": "apps-external/deck", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_deck_app" - }, - { - "name": "end_to_end_encryption", - "path": "apps-external/end_to_end_encryption", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_end_to_end_encryption_app" - }, - { - "name": "forms", - "path": "apps-external/forms", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_forms_app" - }, - { - "name": "groupfolders", - "path": "apps-external/groupfolders", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_groupfolders_app" - }, - { - "name": "integration_openai", - "path": "apps-external/integration_openai", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_integration_openai_app" - }, - { - "name": "mail", - "path": "apps-external/mail", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_mail_app" - }, - { - "name": "ncw_apps_menu", - "path": "apps-external/ncw_apps_menu", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_ncw_apps_menu_app" - }, - { - "name": "ncw_mailtemplate", - "path": "apps-external/ncw_mailtemplate", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_ncw_mailtemplate_app" - }, - { - "name": "notes", - "path": "apps-external/notes", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_notes_app" - }, - { - "name": "notifications", - "path": "apps-external/notifications", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_notifications_app" - }, - { - "name": "notify_push", - "path": "apps-external/notify_push", - "has_npm": false, - "has_composer": true, - "makefile_target": "build_notify_push_app" - }, - { - "name": "password_policy", - "path": "apps-external/password_policy", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_password_policy_app" - }, - { - "name": "richdocuments", - "path": "apps-external/richdocuments", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_richdocuments_app" - }, - { - "name": "spreed", - "path": "apps-external/spreed", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_spreed_app" - }, - { - "name": "tables", - "path": "apps-external/tables", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_tables_app" - }, - { - "name": "tasks", - "path": "apps-external/tasks", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_tasks_app" - }, - { - "name": "text", - "path": "apps-external/text", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_text_app" - }, - { - "name": "twofactor_totp", - "path": "apps-external/twofactor_totp", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_twofactor_totp_app" - }, - { - "name": "user_oidc", - "path": "apps-external/user_oidc", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_user_oidc_app" - }, - { - "name": "viewer", - "path": "apps-external/viewer", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_viewer_app" - }, - { - "name": "whiteboard", - "path": "apps-external/whiteboard", - "has_npm": true, - "has_composer": true, - "makefile_target": "build_whiteboard_app" - } - ]' - - # Validate JSON and output as compact format - if echo "$matrix" | jq empty 2>/dev/null; then - echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT - echo "Matrix configuration set successfully" + # Generate matrix from Makefile - single source of truth + echo "Generating matrix from Makefile..." + matrix_output=$(make -f IONOS/Makefile generate_external_apps_matrix_json 2>&1) + + # Filter out info messages to get just the JSON + # Note: Use same filtering logic as validation script to ensure consistency + if echo "$matrix_output" | grep -q '^\[i\]'; then + matrix=$(echo "$matrix_output" | grep -v '^\[i\]') else - echo "Error: Invalid JSON in matrix configuration" + matrix="$matrix_output" + fi + + # Validate JSON + if ! echo "$matrix" | jq empty 2>/dev/null; then + echo "Error: Generated matrix is not valid JSON" + echo "Output: $matrix_output" exit 1 fi - - name: Validate matrix against Makefile + # Output as compact format + echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT + echo "Matrix generated successfully with $(echo "$matrix" | jq 'length') apps" + + - name: Collect apps and their SHAs for cache-based building + id: detect + env: + GH_TOKEN: ${{ github.token }} + CACHE_VERSION: ${{ env.CACHE_VERSION }} run: | - set +e # Intentionally allow script to continue on error for custom error handling and reporting to GITHUB_STEP_SUMMARY + set -e # Exit on error set -u # Exit on undefined variable + set -o pipefail # Exit if any command in pipeline fails - echo "### 🔍 Matrix Validation" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY + echo "Collecting app SHAs and checking cache status..." + echo "" - # Debug: Check if apps-external exists - echo "Checking apps-external directory..." - if [ ! -d "apps-external" ]; then - echo "❌ **Error:** apps-external directory does not exist!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Directory listing:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - ls -la >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - exit 1 - fi + # Get the matrix from previous step + MATRIX='${{ steps.set_matrix.outputs.matrix }}' - echo "Apps-external directory exists. Listing contents:" - ls -la apps-external/ | head -10 + # Build JSON array for apps that actually need building + APPS_TO_BUILD="[]" + # Build JSON array for apps that are cached and need restoring + APPS_TO_RESTORE="[]" + APPS_CHECKED=0 + APPS_CACHED=0 + APPS_TO_BUILD_COUNT=0 + APPS_SHA_MAP="{}" - # Check if jq is available - echo "Checking if jq is installed..." - if ! command -v jq &> /dev/null; then - echo "❌ **Error:** jq is not installed!" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "jq is required for matrix generation but was not found in PATH." >> $GITHUB_STEP_SUMMARY - exit 1 - fi - echo "jq version: $(jq --version)" + echo "### 📦 Cache Status Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| App | SHA | Cache Key | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----|-----|-----------|--------|" >> $GITHUB_STEP_SUMMARY + + # Iterate through each app in the matrix + while IFS= read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + + APPS_CHECKED=$((APPS_CHECKED + 1)) + + # Get current submodule SHA + if [ -d "$APP_PATH" ]; then + CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") + else + echo "⊘ $APP_NAME - directory not found, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Directory not found |" >> $GITHUB_STEP_SUMMARY + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi - echo "Generating matrix from Makefile..." - # Capture both stdout and stderr separately to better diagnose issues - makefile_output=$(make -f IONOS/Makefile generate_external_apps_matrix_json 2>&1) - makefile_exit_code=$? - - echo "Makefile exit code: ${makefile_exit_code}" - echo "Makefile output length: ${#makefile_output}" - - # Debug: Check if GITHUB_STEP_SUMMARY is set - echo "GITHUB_STEP_SUMMARY: ${GITHUB_STEP_SUMMARY:-NOT SET}" - - # If the Makefile command failed, show the error - if [ ${makefile_exit_code} -ne 0 ]; then - echo "" - echo "=== MAKEFILE ERROR ===" - echo "Exit code: ${makefile_exit_code}" - echo "Output:" - echo "$makefile_output" - echo "=====================" - echo "" - - # Write to summary - echo "❌ **Error:** Makefile command failed with exit code ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Makefile error output" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$makefile_output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY + if [ -z "$CURRENT_SHA" ]; then + echo "⊘ $APP_NAME - not a git repo, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Not a git repo |" >> $GITHUB_STEP_SUMMARY + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi - echo "Error written to summary file: ${GITHUB_STEP_SUMMARY}" - exit 1 - fi + # Add SHA to the map for all apps (regardless of cache status) + APPS_SHA_MAP=$(echo "$APPS_SHA_MAP" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '.[$app] = $sha') + + # Cache key that would be used for this app + # Format: -app-build-- + CACHE_KEY="${CACHE_VERSION}-app-build-${APP_NAME}-${CURRENT_SHA}" + SHORT_SHA="${CURRENT_SHA:0:8}" - # Filter out the info message to get just the JSON - # The Makefile outputs "[i] Generating..." to stderr, but we captured everything with 2>&1 - # So we need to extract just the JSON part - generated_matrix=$(echo "$makefile_output" | grep -v '^\[i\]' || echo "$makefile_output") + echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " - workflow_matrix='${{ steps.set-matrix.outputs.matrix }}' + # Check if cache exists using GitHub CLI + CACHE_EXISTS="false" + if ! CACHE_LIST=$(gh cache list --key "$CACHE_KEY" --json key --jq ".[].key" 2>&1); then + echo "⚠️ Warning: Failed to query cache for $APP_NAME: $CACHE_LIST" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⚠️ Cache check failed - will build |" >> $GITHUB_STEP_SUMMARY + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + if echo "$CACHE_LIST" | grep -q "^${CACHE_KEY}$"; then + CACHE_EXISTS="true" + APPS_CACHED=$((APPS_CACHED + 1)) + echo "✓ cached" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> $GITHUB_STEP_SUMMARY + # Add to restore list with SHA + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" '. + [($app + {sha: $sha})]') + else + echo "⚡ needs build" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> $GITHUB_STEP_SUMMARY + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + fi - # Debug output - echo "Generated matrix length: ${#generated_matrix}" - echo "Workflow matrix length: ${#workflow_matrix}" + done < <(echo "$MATRIX" | jq -c '.[]') - # Show first 200 chars of generated matrix for debugging - if [ -n "$generated_matrix" ]; then - echo "Generated matrix preview: ${generated_matrix:0:200}..." - fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Summary:**" >> $GITHUB_STEP_SUMMARY + echo "- Total apps checked: $APPS_CHECKED" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Apps with cached builds: $APPS_CACHED" >> $GITHUB_STEP_SUMMARY + echo "- 🔨 Apps needing build: $APPS_TO_BUILD_COUNT" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY - # Validate that we got valid JSON - if ! echo "$generated_matrix" | jq empty 2>/dev/null; then - echo "❌ **Error:** Generated matrix is not valid JSON" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Invalid JSON output" >> $GITHUB_STEP_SUMMARY + if [ $APPS_CACHED -gt 0 ] && [ $APPS_CHECKED -gt 0 ]; then + CACHE_HIT_PERCENT=$((APPS_CACHED * 100 / APPS_CHECKED)) + echo "**Cache hit rate: ${CACHE_HIT_PERCENT}%** 🎯" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - exit 1 fi - # Validate that we got data - if [ -z "$generated_matrix" ] || [ -z "$workflow_matrix" ]; then - echo "❌ **Error:** Failed to load matrices" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- Generated matrix empty: $([ -z "$generated_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY - echo "- Workflow matrix empty: $([ -z "$workflow_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY - echo "- Makefile exit code: ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY + echo "" + echo "Summary:" + echo " Total apps: $APPS_CHECKED" + echo " Cached: $APPS_CACHED" + echo " To build: $APPS_TO_BUILD_COUNT" + # Validate that we built valid JSON + if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_build" + echo "Content: $APPS_TO_BUILD" exit 1 fi - # Sort both matrices for comparison - generated_sorted=$(echo "$generated_matrix" | jq -S '.' 2>&1 || echo "ERROR") - workflow_sorted=$(echo "$workflow_matrix" | jq -S '.' 2>&1 || echo "ERROR") - - echo "Sorted matrix lengths - generated: ${#generated_sorted}, workflow: ${#workflow_sorted}" + if ! echo "$APPS_TO_RESTORE" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_restore" + echo "Content: $APPS_TO_RESTORE" + exit 1 + fi - # Compare the two matrices - if [ "$generated_sorted" = "$workflow_sorted" ]; then - echo "✅ **Validation passed!** The workflow matrix matches the Makefile configuration." >> $GITHUB_STEP_SUMMARY - echo "" - echo "✅ Matrix validation passed!" + # Output app list with SHAs for the build job to use + # Use proper multiline output format for GitHub Actions + echo "apps_to_build<> $GITHUB_OUTPUT + echo "$APPS_TO_BUILD" >> $GITHUB_OUTPUT + echo "APPS_TO_BUILD_JSON_EOF" >> $GITHUB_OUTPUT + + # Output the list of apps to restore from cache + echo "apps_to_restore<> $GITHUB_OUTPUT + echo "$APPS_TO_RESTORE" >> $GITHUB_OUTPUT + echo "APPS_TO_RESTORE_JSON_EOF" >> $GITHUB_OUTPUT + + # Output the SHA map for all apps + echo "apps_sha_map<> $GITHUB_OUTPUT + echo "$APPS_SHA_MAP" >> $GITHUB_OUTPUT + echo "APPS_SHA_MAP_JSON_EOF" >> $GITHUB_OUTPUT + + # Determine if there are cached apps by comparing counts + # If apps_to_build count is less than total apps, some are cached + if [ $APPS_TO_BUILD_COUNT -lt $APPS_CHECKED ]; then + echo "has_cached_apps=true" >> $GITHUB_OUTPUT else - echo "❌ **Validation failed!** The workflow matrix does not match the Makefile configuration." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - echo "Starting detailed comparison..." - - # Extract app names from both matrices - generated_apps=$(echo "$generated_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") - workflow_apps=$(echo "$workflow_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") - - echo "Generated apps count: $(echo "$generated_apps" | wc -l)" - echo "Workflow apps count: $(echo "$workflow_apps" | wc -l)" - - # Find missing apps (in Makefile but not in workflow) - missing_apps=$(comm -23 <(echo "$generated_apps") <(echo "$workflow_apps")) - if [ $? -ne 0 ]; then - echo "Error: comm command failed when finding missing apps." >&2 - exit 1 - fi - # Find extra apps (in workflow but not in Makefile) - extra_apps=$(comm -13 <(echo "$generated_apps") <(echo "$workflow_apps")) - if [ $? -ne 0 ]; then - echo "Error: comm command failed when finding extra apps." >&2 - exit 1 - fi - - echo "Missing apps: ${missing_apps:-none}" - echo "Extra apps: ${extra_apps:-none}" - - if [ -n "$missing_apps" ]; then - echo "#### ⚠️ Missing Apps" >> $GITHUB_STEP_SUMMARY - echo "The following apps are configured in the Makefile but missing from the workflow:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$missing_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - - if [ -n "$extra_apps" ]; then - echo "#### ⚠️ Extra Apps" >> $GITHUB_STEP_SUMMARY - echo "The following apps are in the workflow but not configured in the Makefile:" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$extra_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - - # Check for configuration mismatches in common apps - common_apps=$(comm -12 <(echo "$generated_apps") <(echo "$workflow_apps") 2>/dev/null || echo "") - - echo "Common apps count: $(echo "$common_apps" | wc -l)" - - if [ -n "$common_apps" ]; then - mismatched_apps="" - - while IFS= read -r app; do - [ -z "$app" ] && continue - gen_config=$(echo "$generated_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") - wf_config=$(echo "$workflow_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") - - if [ -n "$gen_config" ] && [ -n "$wf_config" ] && [ "$gen_config" != "$wf_config" ]; then - mismatched_apps="${mismatched_apps}${app}"$'\n' - fi - done <<< "$common_apps" - - echo "Mismatched apps: ${mismatched_apps:-none}" - - if [ -n "$mismatched_apps" ]; then - echo "#### ⚠️ Configuration Mismatches" >> $GITHUB_STEP_SUMMARY - echo "The following apps have different configurations (has_npm, has_composer, etc.):" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "$mismatched_apps" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "📋 Detailed differences" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - - while IFS= read -r app; do - [ -z "$app" ] && continue - echo "=== $app ===" >> $GITHUB_STEP_SUMMARY - diff -u --label "Workflow" --label "Makefile" \ - <(echo "$workflow_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ - <(echo "$generated_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ - >> $GITHUB_STEP_SUMMARY 2>&1 || true - done <<< "$mismatched_apps" - - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - fi - - # Provide fix instructions - echo "#### 🔧 How to Fix" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Run this command locally to generate the correct matrix:" >> $GITHUB_STEP_SUMMARY - echo '```bash' >> $GITHUB_STEP_SUMMARY - echo "make -f IONOS/Makefile generate_external_apps_matrix_json" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Then update the \`matrix\` variable in \`.github/workflows/build-artifact.yml\` in the set-matrix step with the generated output." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY + echo "has_cached_apps=false" >> $GITHUB_OUTPUT + fi - # Show full diff in expandable section - echo "
" >> $GITHUB_STEP_SUMMARY - echo "📄 Full matrix comparison" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Workflow Matrix:**" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - echo "$workflow_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$workflow_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Makefile Matrix:**" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - echo "$generated_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - echo "" - echo "❌ ERROR: Matrix validation failed!" - echo "See the job summary for details on what's wrong and how to fix it." - echo "Summary file size: $(wc -c < $GITHUB_STEP_SUMMARY || echo 0) bytes" - exit 1 + echo "" + if [ $APPS_TO_BUILD_COUNT -eq 0 ]; then + echo "🎉 All apps are cached! No builds needed." + else + echo "✓ Will build $APPS_TO_BUILD_COUNT app(s)" fi build-external-apps: runs-on: ubuntu-latest needs: prepare-matrix + # Only run if there are apps to build + if: needs.prepare-matrix.outputs.apps_to_build != '[]' permissions: contents: read @@ -523,25 +268,54 @@ jobs: strategy: max-parallel: 20 matrix: - app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + # Use the filtered list of apps that need building (not in cache) + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_build) }} steps: + - name: Get app configuration from full matrix + id: app-config + run: | + # Get the full matrix to look up app configuration + FULL_MATRIX='${{ needs.prepare-matrix.outputs.external_apps_matrix }}' + APP_NAME='${{ matrix.app_info.name }}' + + # Find the app configuration in the full matrix + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') + + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find configuration for $APP_NAME" + exit 1 + fi + + # Extract configuration values + APP_PATH=$(echo "$APP_CONFIG" | jq -r '.path') + HAS_NPM=$(echo "$APP_CONFIG" | jq -r '.has_npm') + HAS_COMPOSER=$(echo "$APP_CONFIG" | jq -r '.has_composer') + MAKEFILE_TARGET=$(echo "$APP_CONFIG" | jq -r '.makefile_target') + + # Set outputs + echo "path=$APP_PATH" >> $GITHUB_OUTPUT + echo "has-npm=$HAS_NPM" >> $GITHUB_OUTPUT + echo "has-composer=$HAS_COMPOSER" >> $GITHUB_OUTPUT + echo "makefile-target=$MAKEFILE_TARGET" >> $GITHUB_OUTPUT + echo "Building $APP_NAME from $APP_PATH (SHA: ${{ matrix.app_info.sha }})" + - name: Checkout server uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 - name: Set up node with version from package.json's engines - if: matrix.app.has_npm - uses: actions/setup-node@v5 + if: steps.app-config.outputs.has-npm == 'true' + uses: actions/setup-node@v6 with: node-version-file: "package.json" cache: 'npm' - cache-dependency-path: ${{ matrix.app.path }}/package-lock.json + cache-dependency-path: ${{ steps.app-config.outputs.path }}/package-lock.json - name: Setup PHP with PECL extension - if: matrix.app.has_composer + if: steps.app-config.outputs.has-composer == 'true' uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: tools: composer:v2 @@ -549,20 +323,87 @@ jobs: env: runner: ubuntu-latest - - name: Cache Composer dependencies for ${{ matrix.app.name }} - if: matrix.app.has_composer + - name: Cache Composer dependencies for ${{ matrix.app_info.name }} + if: steps.app-config.outputs.has-composer == 'true' uses: actions/cache@v4 with: - path: ${{ matrix.app.path }}/vendor - key: ${{ runner.os }}-composer-${{ matrix.app.name }}-${{ hashFiles(format('{0}/composer.lock', matrix.app.path)) }} + path: ${{ steps.app-config.outputs.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }} restore-keys: | - ${{ runner.os }}-composer-${{ matrix.app.name }}- + ${{ runner.os }}-composer-${{ matrix.app_info.name }}- - - name: Build ${{ matrix.app.name }} app - run: make -f IONOS/Makefile ${{ matrix.app.makefile_target }} + - name: Build ${{ matrix.app_info.name }} app + run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} + + - name: Report build completion + if: success() + run: | + echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ matrix.app_info.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Save built app to cache for future runs + - name: Save build artifacts to cache + uses: actions/cache/save@v4 + with: + path: ${{ steps.app-config.outputs.path }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} - - name: Upload ${{ matrix.app.name }} build artifacts - uses: actions/upload-artifact@v4 + - name: Upload ${{ matrix.app_info.name }} build artifacts + uses: actions/upload-artifact@v5 + with: + retention-days: 1 + name: external-app-build-${{ matrix.app_info.name }} + path: | + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules + + restore-cached-apps: + runs-on: ubuntu-latest + needs: prepare-matrix + # Only run if there are cached apps to restore + if: needs.prepare-matrix.outputs.apps_to_restore != '[]' + + permissions: + contents: read + + name: restore-cached-apps + strategy: + max-parallel: 20 + matrix: + # Use the filtered list of apps that need restoring from cache + app: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_restore) }} + + steps: + - name: Restore cached build from cache + uses: actions/cache/restore@v4 + with: + path: ${{ matrix.app.path }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app.name }}-${{ matrix.app.sha }} + fail-on-cache-miss: true + + - name: Validate cached build + run: | + APP_PATH="${{ matrix.app.path }}" + + # Check that the directory exists and is not empty + if [ ! -d "$APP_PATH" ] || [ -z "$(ls -A $APP_PATH)" ]; then + echo "❌ Cache validation failed: Directory is empty or missing" + exit 1 + fi + + # Check for appinfo/info.xml (required for all Nextcloud apps) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Cache validation failed: Missing appinfo/info.xml" + exit 1 + fi + + echo "✅ Cache validation passed for ${{ matrix.app.name }}" + + - name: Upload cached ${{ matrix.app.name }} build artifacts + uses: actions/upload-artifact@v5 with: retention-days: 1 name: external-app-build-${{ matrix.app.name }} @@ -572,7 +413,13 @@ jobs: build-artifact: runs-on: ubuntu-latest - needs: [prepare-matrix, build-external-apps] + needs: [prepare-matrix, build-external-apps, restore-cached-apps] + # Always run this job, even if restore-cached-apps is skipped + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') permissions: contents: read @@ -586,10 +433,10 @@ jobs: uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 - name: Download build external apps - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: pattern: external-app-build-* path: apps-external/ @@ -639,7 +486,7 @@ jobs: done - name: Set up node with version from package.json's engines - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version-file: "package.json" cache: 'npm' @@ -693,7 +540,7 @@ jobs: echo "NC_VERSION=$NC_VERSION" >> $GITHUB_OUTPUT - name: Upload artifact result for job build-artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: retention-days: 30 name: nextcloud_workspace_build_artifact @@ -709,10 +556,16 @@ jobs: upload-to-artifactory: runs-on: self-hosted # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches - if: github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' + if: | + always() && + (github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') && + needs.build-artifact.result == 'success' name: Push to artifactory - needs: [prepare-matrix, build-external-apps, build-artifact] + needs: [prepare-matrix, build-external-apps, restore-cached-apps, build-artifact] outputs: ARTIFACTORY_LAST_BUILD_PATH: ${{ steps.artifactory_upload.outputs.ARTIFACTORY_LAST_BUILD_PATH }} @@ -750,7 +603,7 @@ jobs: fi - name: Download artifact zip - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: nextcloud_workspace_build_artifact @@ -788,12 +641,39 @@ jobs: export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" - # Promote current build to the "latest" dev build - jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ - --build-name "${{ env.BUILD_NAME }}" \ - --build-number ${{ github.run_number }} \ - --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ - $PATH_TO_LATEST_ARTIFACT + # Upload with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + UPLOAD_SUCCESS=false + DELAY_SEC=10 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Upload attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ + $PATH_TO_LATEST_ARTIFACT; then + UPLOAD_SUCCESS=true + echo "✅ Upload successful on attempt $ATTEMPT" + break + else + echo "⚠️ Upload attempt $ATTEMPT failed" + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting $DELAY_SEC seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) # Exponential backoff: delays are 10s, then 20s (sleep occurs after failed attempts) + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$UPLOAD_SUCCESS" != "true" ]; then + echo "❌ Upload failed after $MAX_ATTEMPTS attempts" + exit 1 + fi echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT @@ -806,17 +686,24 @@ jobs: nextcloud-workspace-artifact-to-ghcr_io: runs-on: ubuntu-latest + # Only run if build-artifact succeeded + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') && + needs.build-artifact.result == 'success' permissions: contents: read packages: write name: Push artifact to ghcr.io - needs: [prepare-matrix, build-external-apps, build-artifact] + needs: [prepare-matrix, build-external-apps, restore-cached-apps, build-artifact] steps: - name: Download artifact zip - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: nextcloud_workspace_build_artifact @@ -868,9 +755,14 @@ jobs: runs-on: self-hosted name: Trigger remote workflow - needs: [ build-artifact, upload-to-artifactory ] + needs: [prepare-matrix, build-artifact, upload-to-artifactory] # Trigger remote build on "ionos-dev|ionos-stable" branch *push* defined in the on:push:branches - if: github.event_name == 'push' && ( github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' ) + if: | + github.event_name == 'push' && + (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.prepare-matrix.result == 'success' && + needs.build-artifact.result == 'success' && + needs.upload-to-artifactory.result == 'success' steps: - name: Trigger remote workflow run: | @@ -887,22 +779,55 @@ jobs: BUILD_TYPE="stable" fi - # Trigger GitLab pipeline via webhook with build artifacts and metadata - # Passes GitHub context variables to remote GitLab workflow - curl \ - --silent \ - --insecure \ - --request POST \ - --fail-with-body \ - -o response.json \ - --form token=${{ secrets.GITLAB_TOKEN }} \ - --form ref="stable" \ - --form "variables[GITHUB_SHA]=${{ github.sha }}" \ - --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ - --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ - --form "variables[BUILD_ID]=${{ github.run_id }}" \ - --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ - "${{ secrets.GITLAB_TRIGGER_URL }}" || ( RETCODE="$?"; jq . response.json; exit "$RETCODE" ) + # Trigger GitLab pipeline via webhook with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + TRIGGER_SUCCESS=false + DELAY_SEC=5 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Trigger attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if curl \ + --silent \ + --insecure \ + --request POST \ + --fail-with-body \ + -o response.json \ + --form token=${{ secrets.GITLAB_TOKEN }} \ + --form ref="stable" \ + --form "variables[GITHUB_SHA]=${{ github.sha }}" \ + --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ + --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ + --form "variables[BUILD_ID]=${{ github.run_id }}" \ + --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + "${{ secrets.GITLAB_TRIGGER_URL }}"; then + TRIGGER_SUCCESS=true + echo "✅ Trigger successful on attempt $ATTEMPT" + break + else + RETCODE="$?" + echo "⚠️ Trigger attempt $ATTEMPT failed with code $RETCODE" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting ${DELAY_SEC} seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) # Exponential backoff: 5s, 10s, 20s + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$TRIGGER_SUCCESS" != "true" ]; then + echo "❌ Trigger failed after $MAX_ATTEMPTS attempts" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + exit 1 + fi # Disable command echo set +x