diff --git a/.github/actions/create-test-instance-bullseye/action.yml b/.github/actions/create-test-instance-bullseye/action.yml index 07a8bff2e..dd82d8118 100644 --- a/.github/actions/create-test-instance-bullseye/action.yml +++ b/.github/actions/create-test-instance-bullseye/action.yml @@ -13,7 +13,7 @@ inputs: server_type: description: Server type to use for hetzner servers required: true - default: "cx11" + default: "cx22" outputs: server_address: diff --git a/.github/actions/create-test-instance/action.yml b/.github/actions/create-test-instance/action.yml index ff6cdb9c8..26b1a7303 100644 --- a/.github/actions/create-test-instance/action.yml +++ b/.github/actions/create-test-instance/action.yml @@ -13,7 +13,7 @@ inputs: server_type: description: Server type to use for hetzner servers required: true - default: "cx11" + default: "cx22" outputs: server_address: diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 1d9404856..c5b2992a1 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -348,8 +348,8 @@ jobs: VERSION: "${{ inputs.git_ref || github.ref }}" LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}" - PREVIOUS_IMAGE_URL_ARM64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_arm64_v1.53.2.tar.gz" - PREVIOUS_IMAGE_URL_AMD64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz" + PREVIOUS_IMAGE_URL_ARM64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextcloudPi_LXD_arm64_v1.53.0.tar.gz" + PREVIOUS_IMAGE_URL_AMD64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextCloudPi_LXD_x86_v1.53.0.tar.gz" USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" steps: - name: Setup incus @@ -362,7 +362,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 with: - ref: "${{ env.VERSION }}" + ref: "v1.54.3" - name: Setup Firefox continue-on-error: true id: setup-firefox-browser-action @@ -422,7 +422,7 @@ jobs: sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true exit 1 } - python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { + python nextcloud_tests.py --skip-release-check --no-gui "nextcloudpi.local" 443 4443 || { echo "Nextcloud test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true @@ -457,7 +457,11 @@ jobs: echo "Running update to ${VERSION}" current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" - latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" + latest_nc_version="27.1.6" + + sudo "$LXC" exec ncp -- apt-get update ||: + sudo "$LXC" exec ncp -- apt-get install --no-install-recommends -y gnupg2 + sudo "$LXC" exec ncp -- apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" sudo "$LXC" exec ncp -- /usr/local/bin/ncc status @@ -467,6 +471,7 @@ jobs: echo "Nextcloud is up to date - skipping NC update test." else sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status fi sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log @@ -506,6 +511,10 @@ jobs: echo "System test failed!" exit 1 } + - name: Checkout current version + run: | + git fetch origin + git checkout "${{ env.VERSION }}" - name: NCP distupgrade id: distupgrade run: | @@ -514,10 +523,38 @@ jobs: echo "can't upgrade from Debian $(sudo "$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)" exit 1 } + current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" + sudo "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status - sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + echo "Nextcloud is up to date - skipping NC update test." + else + + for i in {1..10}; + do + echo "running nc update ($i/10)..." + sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status + current_nc_version_new="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]] + then + break + fi + if [[ "$current_nc_version" == "$current_nc_version_new" ]] + then + echo "failed to update to $latest_nc_version" + exit 1 + fi + current_nc_version="$current_nc_version_new" + done + fi + sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log sudo "$LXC" stop ncp - name: Relaunch container run: | diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 9aaebb2e5..12a4490d0 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -64,29 +64,28 @@ jobs: # update-binfmts --display qemu-aarch64 # update-binfmts --display qemu-arm - name: "Build Armbian" - if: ${{ inputs.board_id != 'raspberrypi' }} id: build-armbian continue-on-error: true run: | set -ex export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" [[ "${{ github.ref_protected }}" == true ]] || export DBG=x - + ./build/build-SD-armbian.sh "${{ inputs.board_id }}" "${{ inputs.board_name }}" - + artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Build Armbian (2nd attempt)" - if: ${{ inputs.board_id != 'raspberrypi' && steps.build-armbian.outcome == 'failure' }} + if: ${{ steps.build-armbian.outcome == 'failure' }} id: build-armbian-2nd run: | set -ex echo -e "${LOG_CICD} Cleanup armbian build leftovers..." sudo rm -rf armbian/ tmp/ output/ - + export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" [[ "${{ github.ref_protected }}" == true ]] || export DBG=x @@ -98,34 +97,11 @@ jobs: echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Upload Armbian logs" - if: ${{ inputs.board_id != 'raspberrypi' && failure() }} + if: ${{ failure() }} uses: actions/upload-artifact@v3 with: name: ${{ github.run_id }}-${{ inputs.board_id }}-logs path: armbian/output - - name: Build RPI SD Image - if: ${{ inputs.board_id == 'raspberrypi' }} - id: build-rpi - run: | - set -ex - echo -e "${LOG_CICD} Protected? ${{ github.ref_protected }}" - export IMG="NextcloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img" - [[ "${{ github.ref_protected }}" == true ]] || export DBG=x - wget -q https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O ./qemu-aarch64-static - ./build/build-SD-rpi.sh - mkdir -p output - mv "tmp/$IMG" ./output/ - - for i in {1..10} - do - sudo losetup | grep "${IMG}" || break; - [[ "$i" -lt 10 ]] || { echo -e "${LOG_CICD} Timeout while waiting for image to unmount"; exit 1; } - sleep 6 - echo -e "${LOG_CICD} Retrying ($i out of 10)" - done - - echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT - echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: upload image to artifact store uses: actions/upload-artifact@v3 with: @@ -135,7 +111,7 @@ jobs: test: needs: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: VERSION: "${{ inputs.git_ref }}" ARTIFACT_ID: ${{ needs.build.outputs.artifact_name }} @@ -151,18 +127,9 @@ jobs: steps: - name: Set up QEMU uses: docker/setup-qemu-action@v3 -# - name: Set up QEMU -# run: | -# sudo apt-get update -# sudo apt-get install -y binfmt-support -# docker run --rm --privileged tonistiigi/binfmt:latest --install all -# docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes -# sudo systemctl disable apparmor - # sudo mkdir -p /etc/binfmt - # for conf in qemu-{aarch64,arm}-static.conf - # do - # sed 's/:F$/:OC/' /usr/lib/binfmt.d/$conf | sudo tee /etc/binfmt/$conf - # done + - name: Disable apparmor + run: | + sudo systemctl disable apparmor - name: Checkout code uses: actions/checkout@v3 with: @@ -191,9 +158,9 @@ jobs: echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf - sudo mkdir -p raspbian_root/etc/systemd/system/php8.1-fpm.service.d - echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf - echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf + sudo mkdir -p raspbian_root/etc/systemd/system/php8.3-fpm.service.d + echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.3-fpm.service.d/ncp.conf + echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.3-fpm.service.d/ncp.conf - name: Test image id: test run: | @@ -208,13 +175,13 @@ jobs: sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" & sleep 60 - CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) + CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) success=false for attempt in {1..30} do echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) ==" - ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" + ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; } success=true break @@ -249,12 +216,14 @@ jobs: done [[ "$success" == "true" ]] || { - echo -e "${LOG_CICD} Timeout reached." - "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" - "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" + echo -e "${LOG_CICD} ERR: Timeout reached." + "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q systemctl status redis-server |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + sudo journalctl --file ./raspbian_root/var/log/journal/"$(sudo cat ./raspbian_root/etc/machine-id)"/system.journal --no-pager -eu redis-server ||: + "${CONTAINER_CMD[@]}" -q journalctl --no-pager -eu redis-server + "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: + "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" ||: exit 1 } @@ -310,15 +279,24 @@ jobs: python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} Nextcloud test failed!" + echo -e "${LOG_DIAG} /etc/os-release:" + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /etc/os-release' + echo -e "${LOG_DIAG} /usr/local/etc/ncp.cfg:" + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'cat /usr/local/etc/ncp.cfg' + cat ./raspbian_root/usr/local/etc/ncp.cfg + echo -e "${LOG_DIAG} /home/ncp-app-bridge config ncp" + sudo ls -l ./raspbian_root/home/www/ncp-app-bridge.sh + "${CONTAINER_CMD[@]}" --pipe --uid=33 ncp /bin/bash -c 'sudo /home/www/ncp-app-bridge.sh config ncp' echo -e "{$LOG_DIAG} Geckodriver logs:" tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo -e "${LOG_CICD} ================" echo -e "${LOG_DIAG} ncp.log: " - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo "================" echo "${LOG_DIAG} Nextcloud log: " - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true - "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" --pipe -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + sudo cat ./raspbian_root/opt/ncdata/data/nextcloud.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" sleep 12 continue } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b92f3cf95..b6ce6a4f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -136,6 +136,22 @@ jobs: board_id: odroidc2 board_name: OdroidC2 secrets: inherit + orangepi5: + if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + board_id: orangepi5 + board_name: OrangePi5 + secrets: inherit + orangepi5plus: + if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + board_id: orangepi5-plus + board_name: OrangePi5Plus + secrets: inherit lxd-x86-release: needs: @@ -294,6 +310,30 @@ jobs: artifact_file: "${{ needs.odroidc2.outputs.artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} secrets: inherit + orangepi5-release: + needs: + - orangepi5 + - github-release + if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.orangepi5.outputs.artifact_name }}" + artifact_file: "${{ needs.orangepi5.outputs.artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + secrets: inherit + orangepi5plus-release: + needs: + - orangepi5plus + - github-release + if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.orangepi5plus.outputs.artifact_name }}" + artifact_file: "${{ needs.orangepi5plus.outputs.artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + secrets: inherit github-release: if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(github.ref_name, 'docker-') }} @@ -324,6 +364,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + + NC_VERSION="$(jq -r '.nextcloud_version' < etc/ncp.cfg)" + PHP_VERSION="$(jq -r '.php_version' < etc/ncp.cfg)" + DEBIAN_VERSION="$(jq -r '.release' < etc/ncp.cfg)" + ARMBIAN_VERSIOn="$(cat build/armbian/armbian_version)" + subject="$(git tag -n10 --format="%(contents:subject)" "${{ env.VERSION }}")" body="$(git tag -n30 --format="%(contents:body)" "${{ env.VERSION }}")" separator=" @@ -334,6 +380,12 @@ jobs: gh release create --draft -F - "${{ env.VERSION }}" <> "$GITHUB_ENV" @@ -306,13 +307,14 @@ jobs: name: "${{ env.SSH_ARTIFACT_NAME }}" path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error + include-hidden-files: true - id: create-test-instance uses: ./.github/actions/create-test-instance-bullseye with: version: "${{ env.PREVIOUS_VERSION }}" uid: "${{ env.UID }}" hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} - server_type: "cx11" + server_type: "cx22" - name: Set instance variables run: | echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" @@ -339,7 +341,7 @@ jobs: echo "Run integration tests" ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg - test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + test-ncp-instance -a -f "$SNAPSHOT_ID" -b "v1.54.3" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" @@ -385,7 +387,7 @@ jobs: NC_TEST_ARGS=() ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check") set -x - test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + test-ncp-instance -f "$SNAPSHOT_ID" -b "v1.54.3" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" @@ -418,6 +420,45 @@ jobs: } ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade echo "skipped=no" | tee -a $GITHUB_OUTPUT + - name: Update Nextcloud + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin + run: | + set -ex + apk add jq + + echo "Setup ssh" + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + current_nc_version="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')" + latest_nc_version="$(cat ../../etc/ncp.cfg | jq -r '.nextcloud_version')" + + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + echo "Nextcloud is up to date - skipping NC update test." + else + for i in {1..10}; + do + echo "running nc update ($i/10)..." + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "bash -c 'DBG=x ncp-update-nc ${latest_nc_version?}'" + + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" /usr/local/bin/ncc status + + current_nc_version_new="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')" + if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]] + then + break + fi + if [[ "$current_nc_version" == "$current_nc_version_new" ]] + then + echo "failed to update to $latest_nc_version" + exit 1 + fi + current_nc_version="$current_nc_version_new" + done + fi - name: Run integration tests after dist-upgrade id: final_test diff --git a/README.md b/README.md index c35eebc19..00980c7d0 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Please reach out in the [Matrix][chat-matrix-wiki] or [Telegram][chat-telegram-w ## Features - * Raspberry Pi OS/Debian 11 _(Bullseye)_ + * Raspberry Pi OS/Debian 12 _(Bookworm)_ * Nextcloud * Apache, with HTTP2 enabled * PHP 8.1 diff --git a/bin/ncp-check-nc-version b/bin/ncp-check-nc-version index a89c63273..972a5b8a4 100755 --- a/bin/ncp-check-nc-version +++ b/bin/ncp-check-nc-version @@ -7,7 +7,7 @@ set -e source /usr/local/etc/library.sh # sets NCLATESTVER CURRENT="$(nc_version)" -NEXT_VERSION="$(determine_nc_upgrade_version "${CURRENT}" "${NCLATESTVER?}")" +NEXT_VERSION="$(determine_nc_update_version "${CURRENT}" "${NCLATESTVER?}")" [[ -n "$NEXT_VERSION" ]] || exit 0 NOTIFIED=/var/run/.nc-version-notified diff --git a/bin/ncp-update-nc b/bin/ncp-update-nc index 05b1c01f8..337cc8c84 100755 --- a/bin/ncp-update-nc +++ b/bin/ncp-update-nc @@ -19,303 +19,48 @@ set -eE${DBG} -VER="$1" BIN="${0##*/}" - -source /usr/local/etc/library.sh - -# pre-checks -#################### +VER="$1" [[ "$VER" == "" ]] && { echo "Usage ${BIN} "; exit 1; } -[[ -f /.docker-image ]] && BASEDIR=/data || BASEDIR=/var/www -cd "$BASEDIR" -DATADIR="$( get_nc_config_value datadirectory )" -ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Nextcloud is currently down"; exit 1; } -[[ -d "${BASEDIR}/nextcloud-old" ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; } -[[ -d "${BASEDIR}/nextcloud" ]] || { echo "Nextcloud directory not found" ; exit 1; } -[[ -d "$DATADIR" ]] || { echo "Nextcloud data directory not found" ; exit 1; } -# check version -#################### - -[[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" -CURRENT="$(nc_version)" -TARGET_VERSION="$(determine_nc_upgrade_version "${CURRENT?}" "${VER?}")" -[[ "$TARGET_VERSION" == "$CURRENT" ]] && { - echo "Nextcloud version ${CURRENT} is already installed. Nothing to do." - exit 0 -} -[[ -n "$TARGET_VERSION" ]] || { - echo "Could not find a valid upgrade path from '${CURRENT}' to '${TARGET_VERSION}'. Nothing to update." - exit 1 +connect_to_nc_update() { + tail -n 100 -f "/var/log/ncp-update-nc.log" & + tail_pid=$! + while [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] + do + sleep 3 + done + + kill "$tail_pid" + if [[ "$(systemctl is-active ncp-update-nc ||:)" == "inactive" ]] + then + echo "Nextcloud update finished successfully." + return 0 + elif [[ "$(systemctl is-active ncp-update-nc ||:)" == "failed" ]] + then + echo "Nextcloud update failed." + return 1 + else + echo "Nextcloud update was not found or failed (unexpected status: '$(systemctl is-active ncp-update-nc ||:)')" + fi } -MAJOR_NEW="${TARGET_VERSION%%.*}" - -if [[ "$MAJOR_NEW" -ge 24 ]] && [[ "$(lsb_release -r)" =~ .*10 ]] +if [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|activating|deactivating)$ ]] then - echo -e "NCP doesn't support Nextcloud versions greater than 23 with Debian 10 (Buster). Please run ncp-dist-upgrade." - exit 1 -fi - -if [[ "$MAJOR_NEW" -ge 29 ]] && [[ "$(lsb_release -r)" =~ .*12 ]] -then - echo -e "NCP doesn't support Nextcloud versions greater than 28 with Debian 11 (Bullseye). Please run ncp-dist-upgrade." - exit 1 -fi - -grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; } -grep -qP "\d+\.\d+\.\d+" <<<"$TARGET_VERSION" || { echo "Malformed version $TARGET_VERSION" ; exit 1; } - -echo "Current Nextcloud version $CURRENT" -echo "Available Nextcloud version $TARGET_VERSION" -if [[ "$TARGET_VERSION" != "$VER" ]] -then - echo "INFO: You have requested an update to '$VER', but a direct update to '$VER' cannot be performed, so the latest available version that can be updated to has been selected automatically." -fi - -# make sure that cron.php is not running and there are no pending jobs -# https://github.com/nextcloud/server/issues/10949 -pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; } -pgrep -cf cron.php &>/dev/null && { echo "cron.php running. Abort"; exit 1; } -mysql nextcloud <<<"UPDATE ${DB_PREFIX}jobs SET reserved_at=0;" - -# cleanup -#################### -cleanup() { - local RET=$? - set +eE - echo "Clean up..." - rm -rf "$BASEDIR"/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old - trap "" EXIT - exit $RET -} -trap cleanup EXIT - -# get new code -#################### -URL="https://download.nextcloud.com/server/releases/nextcloud-$TARGET_VERSION.tar.bz2" -echo "Download Nextcloud $TARGET_VERSION..." -wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; } - -# backup -#################### -BKPDIR="$BASEDIR" -WITH_DATA=no -COMPRESSED=yes -LIMIT=0 - -echo "Back up current instance..." -set +eE -ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" # && false # test point -RET=$? -sync -set -eE - -BKP_="$( ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -1 )" -[[ -f "$BKP_" ]] || { set +eE; echo "Error backing up"; false || cleanup; } -[[ $RET -ne 0 ]] && { rm -f "$BKP_"; set +eE; echo "Error backing up"; false || cleanup; } -BKP="$( dirname "$BKP_" )/$( basename "$BKP_" .tar.gz )-${CURRENT}.tar.gz" -echo "Storing backup at '$BKP'..." -mv "$BKP_" "$BKP" - -# simple restore if anything fails from here -#################### -rollback_simple() { - set +eE - trap "" INT TERM HUP ERR - echo -e "Abort\nSimple roll back..." - rm -rf "$BASEDIR"/nextcloud - mv "$BASEDIR"/nextcloud-old "$BASEDIR"/nextcloud - false || cleanup # so cleanup exits with 1 -} -trap rollback_simple INT TERM HUP ERR - -# replace code -#################### -echo "Install Nextcloud $TARGET_VERSION..." -mv -T nextcloud nextcloud-old -tar -xf nextcloud.tar.bz2 # && false # test point -rm -rf /var/www/nextcloud.tar.bz2 - -# copy old config -#################### -cp nextcloud-old/config/config.php nextcloud/config/ - -# copy old themes -#################### -cp -raT nextcloud-old/themes/ nextcloud/themes/ - -# copy old NCP apps -#################### -for app in nextcloudpi previewgenerator; do - if [[ -d nextcloud-old/apps/"${app}" ]]; then - cp -r -L nextcloud-old/apps/"${app}" /var/www/nextcloud/apps/ - fi -done - -#false # test point - -# copy data if it was at the default location -#################### -if [[ "$DATADIR" == "/var/www/nextcloud/data" ]] || [[ "$DATADIR" == "/data/nextcloud/data" ]]; then - echo "Restore data..." - mv -T nextcloud-old/data nextcloud/data + echo "Existing ncp-update-nc process detected. Connecting..." + connect_to_nc_update + exit $? fi -# nc-restore if anything fails from here -#################### -rollback() { - set +eE - trap "" INT TERM HUP ERR EXIT - echo -e "Abort\nClean up..." - rm -rf /var/www/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old - echo "Rolling back to backup $BKP..." - local TMPDATA - mkdir -p "$BASEDIR/recovery/" - TMPDATA="$( mktemp -d "$BASEDIR/recovery/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } - [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && mv -T "$DATADIR" "$TMPDATA" - ncp-restore "$BKP" || { echo "Rollback failed! Data left at $TMPDATA"; exit 1; } - [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && { rm -rf "$DATADIR"; mv -T "$TMPDATA" "$DATADIR"; } - rm "$BKP" - echo "Rollback successful. Nothing was updated" - exit 1 -} -trap rollback INT TERM HUP ERR - -# fix permissions -#################### -echo "Fix permissions..." -chown -R www-data:www-data nextcloud -find nextcloud/ -type d -exec chmod 750 {} \; -find nextcloud/ -type f -exec chmod 640 {} \; - -# upgrade -#################### -echo "Upgrade..." -ncc='sudo -u www-data php nextcloud/occ' -$ncc upgrade # && false # test point -$ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n -$ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n -$ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n -$ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n - -# use the correct version for custom apps -NCVER="$(nc_version)" -if is_more_recent_than "21.0.0" "${NCVER}"; then - NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20 -else - # Install notify_push if not installed - if ! is_app_enabled notify_push; then - ncc app:install notify_push - ncc app:enable notify_push - install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf - a2enmod proxy proxy_http proxy_wstunnel - apachectl -k graceful - ## make sure the notify_push daemon is runnnig - - install_template systemd/notify_push.service.sh /etc/systemd/system/notify_push.service - start_notify_push - nc_domain="$(ncc config:system:get overwrite.cli.url)" - set-nc-domain "${nc_domain}" || { - echo "notify_push setup failed. You are probably behind a proxy" - echo "Run 'ncc config:system:set trusted_proxies 15 --value=' and then 'ncc notify_push:setup https:///push to enable" - echo "Check https://help.nextcloud.com/tags/ncp for support" - } - - fi - NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21 -fi -rm -rf /var/www/nextcloud/apps/previewgenerator -ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator +systemctl reset-failed ncp-encrypt ||: +systemd-run -u 'ncp-update-nc' bash -c "DBG='${DBG:-}' /usr/local/bin/ncp-update-nc.d/update-nc.sh '${VER}' |& tee /var/log/ncp-update-nc.log" +sleep 5 -if ! is_docker && ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0" +if ! [[ "$(systemctl is-active ncp-update-nc ||:)" =~ ^(active|inactive|activating|deactivating)$ ]] then - ( - echo "Upgrading PHP..." - export DEBIAN_FRONTEND=noninteractive - PHPVER_OLD="$PHPVER" - PHPVER_NEW="8.1" - PHP_PACKAGES_OLD=(php-{common,igbinary,redis,json} "php${PHPVER_OLD}" \ - "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp}) - PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ - "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) - - php_restore() { - trap "" INT TERM HUP ERR - echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." - set +e - service "php${PHPVER_NEW}-fpm" stop - a2disconf php${PHPVER_NEW}-fpm - rm /etc/apt/sources.list.d/php.list - apt-get update - apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" - apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" - set_ncpcfg "php_version" "${PHPVER_OLD}" - install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" - run_app nc-limits - a2enconf "php${PHPVER_OLD}-fpm" - service "php${PHPVER_OLD}-fpm" start - service apache2 restart - echo "PHP upgrade has been successfully reverted" - set -e - } - - trap php_restore INT TERM HUP ERR - - # Setup apt repository for php 8 - wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg - echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list - apt-get update - - clear_opcache - - echo "Stopping apache and php-fpm..." - service "php${PHPVER_OLD}-fpm" stop - service apache2 stop - - echo "Remove old PHP (${PHPVER_OLD})..." - a2disconf "php${PHPVER_OLD}-fpm" - - apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" - - echo "Install PHP ${PHPVER_NEW}..." - install_with_shadow_workaround --no-install-recommends systemd - apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" - - set_ncpcfg "php_version" "${PHPVER_NEW}" - install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" - ( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) - - a2enconf "php${PHPVER_NEW}-fpm" - - echo "Starting apache and php-fpm..." - service "php${PHPVER_NEW}-fpm" start - service apache2 start - ncc status - ) - - # Reload library.sh to reset PHPVER - source /usr/local/etc/library.sh - + echo "Failed to start ncp-update-nc" + [[ -f /var/log/ncp-update-nc.log ]] && cat /var/log/ncp-update-nc.log + systemctl status --no-pager ncp-update-nc ||: fi -# refresh completions -ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp - -is_docker && { - killall notify_push - sleep 1 - start_notify_push -} - -echo "Update completed successfully." -# done -#################### -mkdir -p "$DATADIR"/ncp-update-backups -mv "$BKP" "$DATADIR"/ncp-update-backups -chown -R www-data:www-data "$DATADIR"/ncp-update-backups -BKP="$DATADIR"/ncp-update-backups/"$( basename "$BKP" )" -echo "Backup stored at $BKP" - -bash -c "sleep 5; source /usr/local/etc/library.sh; clear_opcache;" &>/dev/null & +connect_to_nc_update diff --git a/bin/ncp-update-nc.d/update-nc.sh b/bin/ncp-update-nc.d/update-nc.sh new file mode 100755 index 000000000..1055971bf --- /dev/null +++ b/bin/ncp-update-nc.d/update-nc.sh @@ -0,0 +1,248 @@ +#!/bin/bash + +set -eE${DBG} + +VER="$1" +source /usr/local/etc/library.sh +export RELEASE +export PHPVER + +# pre-checks +#################### +BASEDIR=/var/www +cd "$BASEDIR" +DATADIR="$( get_nc_config_value datadirectory )" +ncc status &>/dev/null || { [[ "$DBG" == x ]] && ncc status; echo "Nextcloud is currently down"; exit 1; } +[[ -d "${BASEDIR}/nextcloud-old" ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; } +[[ -d "${BASEDIR}/nextcloud" ]] || { echo "Nextcloud directory not found" ; exit 1; } +[[ -d "$DATADIR" ]] || { echo "Nextcloud data directory not found" ; exit 1; } + +# check version +#################### + +[[ ${EUID} -eq 0 ]] && SUDO="sudo -u www-data" +CURRENT="$(nc_version)" +if [[ "$VER" == "0" ]] || [[ "$VER" == "" ]] +then + REQUESTED_VERSION="${NCLATESTVER?}" +else + REQUESTED_VERSION="$VER" +fi +TARGET_VERSION="$(determine_nc_update_version "${CURRENT?}" "${NCLATESTVER}" "${REQUESTED_VERSION}")" +[[ "$TARGET_VERSION" == "$CURRENT" ]] && { + echo "Nextcloud version ${CURRENT} is already installed. Nothing to do." + exit 0 +} +[[ -n "$TARGET_VERSION" ]] || { + echo "Could not find a valid upgrade path from '${CURRENT}' to '${TARGET_VERSION}'. Nothing to update." + exit 1 +} + +MAJOR_NEW="${TARGET_VERSION%%.*}" +DEBIAN_VERSION="$(. /etc/os-release; echo "$VERSION_ID")" + +if [[ "$MAJOR_NEW" -ge 24 ]] && [[ $DEBIAN_VERSION -le 10 ]] +then + echo -e "NCP doesn't support Nextcloud versions greater than 23 with Debian 10 (Buster). Please run ncp-dist-upgrade." + exit 1 +fi + +if [[ "$MAJOR_NEW" -ge 29 ]] && [[ $DEBIAN_VERSION -le 11 ]] +then + echo -e "NCP doesn't support Nextcloud versions greater than 28 with Debian 11 (Bullseye). Please run ncp-dist-upgrade." + exit 1 +fi + +grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; } +grep -qP "\d+\.\d+\.\d+" <<<"$TARGET_VERSION" || { echo "Malformed version $TARGET_VERSION" ; exit 1; } + +echo "Current Nextcloud version $CURRENT" +echo "Available Nextcloud version $TARGET_VERSION" +if [[ "$TARGET_VERSION" != "$REQUESTED_VERSION" ]] +then + echo "INFO: You have requested an update to '${REQUESTED_VERSION}', but a direct update to '${REQUESTED_VERSION}' cannot be performed, so the latest available version that can be updated to has been selected automatically." +fi + +# make sure that cron.php is not running and there are no pending jobs +# https://github.com/nextcloud/server/issues/10949 +pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; } +pgrep -cf cron.php &>/dev/null && { echo "cron.php running. Abort"; exit 1; } +mysql nextcloud <<<"UPDATE ${DB_PREFIX}jobs SET reserved_at=0;" + +# cleanup +#################### +cleanup() { + local RET=$? + set +eE + echo "Clean up..." + rm -rf "$BASEDIR"/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old + trap "" EXIT + exit $RET +} +trap cleanup EXIT + +# get new code +#################### +URL="https://download.nextcloud.com/server/releases/nextcloud-$TARGET_VERSION.tar.bz2" +echo "Download Nextcloud $TARGET_VERSION..." +wget -q "$URL" -O nextcloud.tar.bz2 || { echo "Error downloading"; exit 1; } + +# backup +#################### +BKPDIR="$BASEDIR" +WITH_DATA=no +COMPRESSED=yes +LIMIT=0 + +echo "Back up current instance..." +set +eE +ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" # && false # test point +RET=$? +sync +set -eE + +BKP_="$( ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -1 )" +[[ -f "$BKP_" ]] || { set +eE; echo "Error backing up"; false || cleanup; } +[[ $RET -ne 0 ]] && { rm -f "$BKP_"; set +eE; echo "Error backing up"; false || cleanup; } +BKP="$( dirname "$BKP_" )/$( basename "$BKP_" .tar.gz )-${CURRENT}.tar.gz" +echo "Storing backup at '$BKP'..." +mv "$BKP_" "$BKP" + +# simple restore if anything fails from here +#################### +rollback_simple() { + set +eE + trap "" INT TERM HUP ERR + echo -e "Abort\nSimple roll back..." + rm -rf "$BASEDIR"/nextcloud + mv "$BASEDIR"/nextcloud-old "$BASEDIR"/nextcloud + false || cleanup # so cleanup exits with 1 +} +trap rollback_simple INT TERM HUP ERR + +# replace code +#################### +echo "Install Nextcloud $TARGET_VERSION..." +mv -T nextcloud nextcloud-old +tar -xf nextcloud.tar.bz2 # && false # test point +rm -rf /var/www/nextcloud.tar.bz2 + +# copy old config +#################### +cp nextcloud-old/config/config.php nextcloud/config/ + +# copy old themes +#################### +cp -raT nextcloud-old/themes/ nextcloud/themes/ + +# copy old NCP apps +#################### +for app in nextcloudpi previewgenerator; do + if [[ -d nextcloud-old/apps/"${app}" ]]; then + cp -r -L nextcloud-old/apps/"${app}" /var/www/nextcloud/apps/ + fi +done + +#false # test point + +# copy data if it was at the default location +#################### +if [[ "$DATADIR" == "/var/www/nextcloud/data" ]] || [[ "$DATADIR" == "/data/nextcloud/data" ]]; then + echo "Restore data..." + mv -T nextcloud-old/data nextcloud/data +fi + +# nc-restore if anything fails from here +#################### +rollback() { + set +eE + trap "" INT TERM HUP ERR EXIT + echo -e "Abort\nClean up..." + rm -rf /var/www/nextcloud.tar.bz2 "$BASEDIR"/nextcloud-old + echo "Rolling back to backup $BKP..." + local TMPDATA + mkdir -p "$BASEDIR/recovery/" + TMPDATA="$( mktemp -d "$BASEDIR/recovery/ncp-data.XXXXXX" )" || { echo "Failed to create temp dir" >&2; exit 1; } + [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && mv -T "$DATADIR" "$TMPDATA" + ncp-restore "$BKP" || { echo "Rollback failed! Data left at $TMPDATA"; exit 1; } + [[ "$DATADIR" == "$BASEDIR/nextcloud/data" ]] && { rm -rf "$DATADIR"; mv -T "$TMPDATA" "$DATADIR"; } + rm "$BKP" + echo "Rollback successful. Nothing was updated" + exit 1 +} +trap rollback INT TERM HUP ERR + +# fix permissions +#################### +echo "Fix permissions..." +chown -R www-data:www-data nextcloud +find nextcloud/ -type d -exec chmod 750 {} \; +find nextcloud/ -type f -exec chmod 640 {} \; + +# upgrade +#################### +echo "Upgrade..." +ncc='sudo -u www-data php nextcloud/occ' +$ncc upgrade # && false # test point +$ncc | grep -q db:add-missing-indices && $ncc db:add-missing-indices -n +$ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n +$ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n +$ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n +$ncc maintenance:repair --help | grep -q -e '--include-expensive' && $ncc maintenance:repair --include-expensive + +# use the correct version for custom apps +NCVER="$(nc_version)" +if is_more_recent_than "21.0.0" "${NCVER}"; then + NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc20 +else + # Install notify_push if not installed + if ! is_app_enabled notify_push; then + ncc app:install notify_push + ncc app:enable notify_push + install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf + a2enmod proxy proxy_http proxy_wstunnel + apachectl -k graceful + ## make sure the notify_push daemon is runnnig + + install_template systemd/notify_push.service.sh /etc/systemd/system/notify_push.service + start_notify_push + nc_domain="$(ncc config:system:get overwrite.cli.url)" + set-nc-domain "${nc_domain}" || { + echo "notify_push setup failed. You are probably behind a proxy" + echo "Run 'ncc config:system:set trusted_proxies 15 --value=' and then 'ncc notify_push:setup https:///push to enable" + echo "Check https://help.nextcloud.com/tags/ncp for support" + } + + fi + NCPREV=/var/www/ncp-previewgenerator/ncp-previewgenerator-nc21 +fi +rm -rf /var/www/nextcloud/apps/previewgenerator +ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator + +if ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0" +then + /usr/local/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh + + # Reload library.sh to reset PHPVER + source /usr/local/etc/library.sh +elif ! is_more_recent_than "29.0.0" "${NCVER}" && is_more_recent_than "8.3.0" "${PHPVER}.0" && [[ "$DEBIAN_VERSION" -ge 12 ]] +then + /usr/local/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh + + # Reload library.sh to reset PHPVER + source /usr/local/etc/library.sh +fi + +# refresh completions +ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp + +echo "Update completed successfully." +# done +#################### +mkdir -p "$DATADIR"/ncp-update-backups +mv "$BKP" "$DATADIR"/ncp-update-backups +chown -R www-data:www-data "$DATADIR"/ncp-update-backups +BKP="$DATADIR"/ncp-update-backups/"$( basename "$BKP" )" +echo "Backup stored at $BKP" + +bash -c "sleep 5; source /usr/local/etc/library.sh; clear_opcache;" &>/dev/null & diff --git a/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh new file mode 100755 index 000000000..c84c322a8 --- /dev/null +++ b/bin/ncp-update-nc.d/upgrade-php-bookworm-8.3.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +source /usr/local/etc/library.sh + +echo "Upgrading PHP..." +export DEBIAN_FRONTEND=noninteractive +PHPVER_OLD="$PHPVER" +PHPVER_NEW="8.3" +PHP_PACKAGES_OLD=("php${PHPVER_OLD}" \ + "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) +PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ + "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) + +php_restore() { + trap "" INT TERM HUP ERR + echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." + set +e + service "php${PHPVER_NEW}-fpm" stop + a2disconf php${PHPVER_NEW}-fpm + wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg + echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list + apt-get update + apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" + set_ncpcfg "php_version" "${PHPVER_OLD}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + run_app nc-limits + a2enconf "php${PHPVER_OLD}-fpm" + service "php${PHPVER_OLD}-fpm" start + service apache2 restart + echo "PHP upgrade has been successfully reverted" + set -e +} + +trap php_restore INT TERM HUP ERR + +apt-get update + +clear_opcache + +echo "Stopping apache and php-fpm..." +service "php${PHPVER_OLD}-fpm" stop +service apache2 stop + +echo "Remove old PHP (${PHPVER_OLD})..." +a2disconf "php${PHPVER_OLD}-fpm" + +apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" + +echo "Install PHP ${PHPVER_NEW}..." +install_with_shadow_workaround --no-install-recommends systemd +apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" + +set_ncpcfg "php_version" "${PHPVER_NEW}" +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" +( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) + +a2enconf "php${PHPVER_NEW}-fpm" + +[[ -f "/etc/systemd/system/php${PHPVER_OLD}-fpm.service.d/ncp.conf" ]] && { + mkdir -p "/etc/systemd/system/php${PHPVER_NEW}-fpm.service.d" + cp "/etc/systemd/system/php${PHPVER_OLD}-fpm.service.d/ncp.conf" "/etc/systemd/system/php${PHPVER_NEW}-fpm.service.d/ncp.conf" +} + +echo "Starting apache and php-fpm..." +service "php${PHPVER_NEW}-fpm" start +service apache2 start +ncc status \ No newline at end of file diff --git a/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh new file mode 100755 index 000000000..4df0f7a95 --- /dev/null +++ b/bin/ncp-update-nc.d/upgrade-php-bullseye-8.1.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +source /usr/local/etc/library.sh + +echo "Upgrading PHP..." +export DEBIAN_FRONTEND=noninteractive +PHPVER_OLD="$PHPVER" +PHPVER_NEW="8.1" +PHP_PACKAGES_OLD=(php-{common,igbinary,redis,json} "php${PHPVER_OLD}" \ + "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp}) +PHP_PACKAGES_NEW=("php${PHPVER_NEW}" \ + "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) + +php_restore() { + trap "" INT TERM HUP ERR + echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." + set +e + service "php${PHPVER_NEW}-fpm" stop + a2disconf php${PHPVER_NEW}-fpm + rm /etc/apt/sources.list.d/php.list + apt-get update + apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" + set_ncpcfg "php_version" "${PHPVER_OLD}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + run_app nc-limits + a2enconf "php${PHPVER_OLD}-fpm" + service "php${PHPVER_OLD}-fpm" start + service apache2 restart + echo "PHP upgrade has been successfully reverted" + set -e +} + +trap php_restore INT TERM HUP ERR + +# Setup apt repository for php 8 +wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg +echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list +apt-get update + +clear_opcache + +echo "Stopping apache and php-fpm..." +service "php${PHPVER_OLD}-fpm" stop +service apache2 stop + +echo "Remove old PHP (${PHPVER_OLD})..." +a2disconf "php${PHPVER_OLD}-fpm" + +apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" + +echo "Install PHP ${PHPVER_NEW}..." +install_with_shadow_workaround --no-install-recommends systemd +apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" + +set_ncpcfg "php_version" "${PHPVER_NEW}" +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" +( set -e; export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) + +a2enconf "php${PHPVER_NEW}-fpm" + +echo "Starting apache and php-fpm..." +service "php${PHPVER_NEW}-fpm" start +service apache2 start +ncc status \ No newline at end of file diff --git a/bin/ncp/CONFIG/nc-init.sh b/bin/ncp/CONFIG/nc-init.sh index 7892b8e2d..0bc62554c 100644 --- a/bin/ncp/CONFIG/nc-init.sh +++ b/bin/ncp/CONFIG/nc-init.sh @@ -127,11 +127,11 @@ EOF } mysql nextcloud <&1 | sed /^Switch/d + systemctl reset-failed ncp-encrypt ||: + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -fg -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" # switch to the regular virtual hosts after we decrypt, so we can access NC and ncp-web a2ensite ncp 001-nextcloud @@ -70,12 +72,33 @@ configure() mkdir -p "${encdir?}" echo "${PASSWORD?}" | gocryptfs -init -q "${encdir}" save_maintenance_mode + cleanup() { + umount "${datadir}" ||: + [[ -f "${tmpdir}" ]] && { + rm -rf "${datadir?}" ||: + mv "${tmpdir}" "${datadir}" + + chown -R www-data:www-data "${datadir}" + } + } + + trap cleanup 1 trap restore_maintenance_mode EXIT mv "${datadir?}" "${tmpdir?}" mkdir "${datadir}" - echo "${PASSWORD}" | gocryptfs -allow_other -q "${encdir}" "${datadir}" 2>&1 | sed /^Switch/d + systemctl reset-failed ncp-encrypt ||: + systemd-run -u ncp-encrypt -E PASSWORD bash -c "gocryptfs -fg -allow_other -q '${encdir}' '${datadir}' <<<\"\${PASSWORD}\" 2>&1 | sed /^Switch/d |& tee /var/log/ncp-encrypt.log" + + maxtries=5 + while [[ "$(systemctl is-active ncp-encrypt)" != "active" ]] || ! mount | grep -1 "${datadir}" + do + echo "Wating for encryption process to start... (${maxtries})" + sleep 3 + maxtries=$((maxtries - 1)) + [[ $maxtries -gt 0 ]] || return 1 + done echo "Encrypting data..." mv "${tmpdir}"/* "${tmpdir}"/.[!.]* "${datadir}" @@ -85,7 +108,7 @@ configure() set_ncpcfg datadir "${datadir}" echo "Data is now encrypted" -) + } # License diff --git a/bin/ncp/TOOLS/nc-format-USB.sh b/bin/ncp/TOOLS/nc-format-USB.sh index f3483429a..c6ea8979f 100644 --- a/bin/ncp/TOOLS/nc-format-USB.sh +++ b/bin/ncp/TOOLS/nc-format-USB.sh @@ -51,7 +51,7 @@ configure() } DATADIR="$(get_nc_config_value datadirectory || true)" - if [[ $( stat -fc%d / ) != $( stat -fc%d "$DATADIR" ) ]] || [[ -z "$DATADIR" ]] && [[ "$ALLOW_DATA_DIR_REMOVAL" != "yes" ]] + if [[ $( stat -fc%d / ) != $( stat -fc%d "$DATADIR" ) ]] || [[ -z "$DATADIR" ]] && [[ "$ALLOW_DATADIR_REMOVAL" != "yes" ]] then echo "ERROR: Data directory is on USB drive (or can't be determined) and removal of data directory was not explicitly allowed." \ "Please move the data directory to SD before formatting the USB drive." \ diff --git a/bin/ncp/UPDATES/nc-update-nextcloud.sh b/bin/ncp/UPDATES/nc-update-nextcloud.sh index 821bb5e08..58c7487f3 100644 --- a/bin/ncp/UPDATES/nc-update-nextcloud.sh +++ b/bin/ncp/UPDATES/nc-update-nextcloud.sh @@ -12,12 +12,6 @@ LATEST="$NCLATESTVER" configure() { - [[ "$VERSION" == "0" ]] && VERSION="$LATEST" - if ! is_docker && ! is_more_recent_than "24.0.0" "${VERSION}" && is_more_recent_than "8.1.0" "${PHPVER}.0" && [[ " ${BASH_SOURCE[*]} " =~ .*" /home/www/ncp-launcher.sh ".* ]] - then - echo "We need to upgrade PHP. This process cannot be performed from the web UI. Please use 'ncp-config' from the terminal (via SSH or direct access) to update Nextcloud instead. Future updates can again be run from the web UI" - exit 1 - fi bash /usr/local/bin/ncp-update-nc "$VERSION" } diff --git a/build/armbian/armbian.sh b/build/armbian/armbian.sh index 985121062..3d79f4340 100644 --- a/build/armbian/armbian.sh +++ b/build/armbian/armbian.sh @@ -19,6 +19,8 @@ LINUXFAMILY=$2 BOARD=$3 BUILD_DESKTOP=$4 +echo "include home dir? ${INCLUDE_HOME_DIR:-no}" + cd /tmp/overlay NCPCFG=etc/ncp.cfg source etc/library.sh # sets RELEASE diff --git a/build/armbian/armbian_version b/build/armbian/armbian_version new file mode 100644 index 000000000..3e5291183 --- /dev/null +++ b/build/armbian/armbian_version @@ -0,0 +1 @@ +v24.08 \ No newline at end of file diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index 69dae6814..9e4d97174 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -31,7 +31,7 @@ prepare_dirs # tmp cache output # get latest armbian [[ -d armbian ]] || { - git clone --depth 1 --branch v24.08 https://github.com/armbian/build armbian + git clone --depth 1 --branch "$(cat "$(dirname "${0}")/armbian/armbian_version")" https://github.com/armbian/build armbian } #( cd armbian && git pull --ff-only --tags && git checkout v23.02 ) #sed -i -e '/export rootfs_size=/s/du -sm/du --apparent-size -sm/' armbian/lib/functions/image/partitioning.sh diff --git a/changelog.md b/changelog.md index 81ea9eddb..0ad4f656d 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,28 @@ # NextcloudPi Changelog +## [v1.55.0](https://github.com/nextcloud/nextcloudpi/tree/v1.55.0) (2024-09-07) Support for PHP 8.3 and Nextcloud 29 + +### Changes + +- Support for Nextcloud 29 +- Support for PHP 8.3 +- Run `maintenance:repair --include-expensive` after NC update if necessary +- Add logrotate for redis, apache and mysql logs +- Add support for the OrangePi 5 and OrangePi 5 Plus boards + +### Fixes + +- Fix bug in nc-format-USB.sh (thanks @m-breitbach) +- Nextcloud updater fixes + - Fix constant retries of the auto update if on a version that's later than the last officially supported one [#1965](https://github.com/nextcloud/nextcloudpi/issues/1965) + - Fix incorrect detection of os constraints for NC updates beyond v28 [#1960](https://github.com/nextcloud/nextcloudpi/issues/1960) + - Fix user requested version being ignored if a later version was available +- Improve stability of nc-encrypt by daemonizing the encryption process [#1874](https://github.com/nextcloud/nextcloudpi/issues/1874) + +### Development related + +- Perform nc update tests for both lxd as well as vms during CI/CD + ## [v1.54.3](https://github.com/nextcloud/nextcloudpi/tree/v1.54.3) (2024-08-11) Improved Nextcloud (auto) updates ### Fixes diff --git a/etc/library.sh b/etc/library.sh index 1cefac995..74eafcb14 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -519,16 +519,30 @@ function nc_version() ncc status | grep "versionstring:" | awk '{ print $3 }' } -function determine_nc_upgrade_version() { +function determine_nc_update_version() { local current supported current_maj supported_maj versions next_version current="${1?}" supported="${2?}" + requested="${3:-latest}" #CURRENT="$(ncc status | grep "versionstring:" | awk '{ print $3 }')" current_maj="${current%%.*}" + requested_maj="${requested%%.*}" supported_maj="${supported%%.*}" - versions="$(curl -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/nextcloud/server/releases?per_page=100 | jq -r '.[].tag_name' | grep -v -e 'rc.$' -e 'beta.$' | sort -V)" + + # If valid version is requested -> direct update, don't consider anything else + if [[ "$requested" =~ ^[0-9.]*$ ]] && [[ "$requested_maj" -le "$((current_maj + 1))" ]] + then + echo "$requested" + return 0 + fi + + versions="$(curl -q -L -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/nextcloud/server/releases?per_page=100 | jq -r '.[].tag_name' | grep -v -e 'rc.$' -e 'beta.$' | sort -V)" next_version="$(grep "v${current_maj}." <<<"${versions}" | tail -n 1 | tr -d 'v')" + next_version_maj="${next_version%%.*}" + next_version_min="${next_version#*.}" + next_version_min="${next_version_min%%.*}" + if [[ "${next_version}" == "${current}" ]] then diff --git a/etc/ncp-config.d/nc-nextcloud.cfg b/etc/ncp-config.d/nc-nextcloud.cfg index 8f7efa697..a3c428d17 100644 --- a/etc/ncp-config.d/nc-nextcloud.cfg +++ b/etc/ncp-config.d/nc-nextcloud.cfg @@ -9,7 +9,7 @@ { "id": "VER", "name": "Version", - "value": "28.0.7" + "value": "29.0.4" }, { "id": "MAXFILESIZE", diff --git a/etc/ncp.cfg b/etc/ncp.cfg index 86fae7fec..19b3afcdd 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { - "nextcloud_version": "28.0.7", - "php_version": "8.1", + "nextcloud_version": "29.0.4", + "php_version": "8.3", "release": "bookworm" } diff --git a/generate_cohortes.sh b/generate_cohortes.sh index 83fc87b4a..42bd3bd29 100755 --- a/generate_cohortes.sh +++ b/generate_cohortes.sh @@ -5,6 +5,8 @@ target_file="${2?}" all=({100..1}) cohortes=() +touch "${target_file?}" + while IFS="" read -r line || [ -n "$line" ] do [[ -n "$line" ]] || continue diff --git a/ncp-app/appinfo/info.xml b/ncp-app/appinfo/info.xml index f182f0487..fbf207ddb 100644 --- a/ncp-app/appinfo/info.xml +++ b/ncp-app/appinfo/info.xml @@ -12,7 +12,7 @@ tools https://github.com/nextcloud/nextcloudpi/issues - + diff --git a/ncp-app/lib/Service/SettingsService.php b/ncp-app/lib/Service/SettingsService.php index dfc25d3c5..daa78d46a 100644 --- a/ncp-app/lib/Service/SettingsService.php +++ b/ncp-app/lib/Service/SettingsService.php @@ -38,6 +38,7 @@ public function getConfig(string $name, array $defaults): array } if ($config == null) { $this->logger->error("Failed to retrieve ncp config (exit code: $ret)"); + $this->logger->error("ERR: $stderr"); return $defaults; } return $config; @@ -132,11 +133,12 @@ private function runCommand(string $cmd): array { 2 => ["pipe", "w"] ]; - $proc = proc_open($cmd, $descriptorSpec, $pipes, "/home/www-data", null); + $proc = proc_open($cmd, $descriptorSpec, $pipes, "/home/www", null); $stdout = stream_get_contents($pipes[1]); fclose($pipes[1]); $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); + return [proc_close($proc), $stdout, $stderr]; } } diff --git a/ncp-previewgenerator/ncp-previewgenerator-nc21/appinfo/info.xml b/ncp-previewgenerator/ncp-previewgenerator-nc21/appinfo/info.xml index 82fc3a652..8d09f5387 100644 --- a/ncp-previewgenerator/ncp-previewgenerator-nc21/appinfo/info.xml +++ b/ncp-previewgenerator/ncp-previewgenerator-nc21/appinfo/info.xml @@ -25,7 +25,7 @@ The first time you install this app, before using a cron job, you properly want - + diff --git a/ncp.sh b/ncp.sh index 8813c60cd..c2caedcd3 100644 --- a/ncp.sh +++ b/ncp.sh @@ -22,7 +22,8 @@ install() { # NCP-CONFIG apt-get update - $APTINSTALL git dialog whiptail jq file lsb-release tmux + $APTINSTALL git dialog whiptail jq file lsb-release tmux logrotate + mkdir -p "$CONFDIR" "$BINDIR" # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager diff --git a/staged_rollouts/v1.55.0.txt b/staged_rollouts/v1.55.0.txt new file mode 100644 index 000000000..8126544f9 --- /dev/null +++ b/staged_rollouts/v1.55.0.txt @@ -0,0 +1,10 @@ +4 +6 +9 +10 +35 +43 +46 +75 +84 +92 \ No newline at end of file diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index f4fbe209e..dffd4041f 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -27,7 +27,8 @@ from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.firefox.options import Options -from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException, ElementNotInteractableException from typing import List, Tuple import traceback @@ -121,26 +122,37 @@ def is_admin_notifications_checkbox(item: WebElement): return False -def close_first_run_wizard(driver: WebDriver): - wait = WebDriverWait(driver, 60) +def close_first_run_wizard(driver: WebDriver, wait_multiplier=1): + wait = WebDriverWait(driver, 20 * wait_multiplier) first_run_wizard = None try: first_run_wizard = driver.find_element(By.CSS_SELECTOR, "#firstrunwizard") except NoSuchElementException: pass if first_run_wizard is not None and first_run_wizard.is_displayed(): - wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CLASS_NAME, "modal-container__close"), - (By.CLASS_NAME, "first-run-wizard__close-button")])) + for i in range(3): + try: + wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, '.modal-container__content button[aria-label=Close]'), + (By.CLASS_NAME, "modal-container__close"), + (By.CLASS_NAME, "first-run-wizard__close-button")])) + except TimeoutException as e: + if i == 3: + raise e try: - overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") - overlay_close_btn.click() - except NoSuchElementException: - overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") + overlay_close_btn = driver.find_element(By.CSS_SELECTOR, '.modal-container__content button[aria-label=Close]') overlay_close_btn.click() + except (NoSuchElementException, ElementNotInteractableException): + try: + overlay_close_btn = driver.find_element(By.CLASS_NAME, "modal-container__close") + overlay_close_btn.click() + except (NoSuchElementException, ElementNotInteractableException): + overlay_close_btn = driver.find_element(By.CLASS_NAME, "first-run-wizard__close-button") + overlay_close_btn.click() + time.sleep(3) -def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool): +def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool, wait_multiplier=1): """ Login and assert admin page checks""" test = Test() test.new("nextcloud page") @@ -167,7 +179,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: test.report("password", "Wrong password" not in driver.page_source, msg="Failed to login with provided password") test.new("settings config") - wait = WebDriverWait(driver, 60) + wait = WebDriverWait(driver, 60 * wait_multiplier) try: wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, "#security-warning-state-ok"), (By.CSS_SELECTOR, "#security-warning-state-warning"), @@ -186,7 +198,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: elif re.match(r'.*Could not check for JavaScript support.*', warning.text): continue # TODO: Solve redis error logs at the source - elif re.match(r'.*\d+ errors in the logs since.*', warning.text): + elif re.match(r'.*\d+ errors? in the logs since.*', warning.text): continue else: raise ConfigTestFailure(f"WARN: {warning.text}") @@ -224,7 +236,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: except Exception as e: test.check(e) - close_first_run_wizard(driver) + close_first_run_wizard(driver, wait_multiplier) test.new("admin section (1)") try: @@ -277,7 +289,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: except Exception as e: test.check(e) test.new("admin section (2)") - wait = WebDriverWait(driver, 10) + wait = WebDriverWait(driver, 10 * wait_multiplier) try: li = next(filter(is_admin_notifications_checkbox, list_items)) li.find_element(By.TAG_NAME, "input").click() @@ -309,13 +321,14 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: # parse options try: - opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check']) + opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check', 'wait-multiplier=']) except getopt.GetoptError: usage() sys.exit(2) skip_release_check = False options = webdriver.FirefoxOptions() + wait_multiplier = 1 for opt, arg in opts: if opt in ('-h', '--help'): usage() @@ -327,6 +340,8 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: options.add_argument("-headless") elif opt == '--skip-release-check': skip_release_check = True + elif opt == '--wait-multiplier': + wait_multiplier = int(arg) else: usage() sys.exit(2) @@ -365,7 +380,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: driver = webdriver.Firefox(options=options) failed=False try: - test_nextcloud(IP, nc_port, driver, skip_release_check) + test_nextcloud(IP, nc_port, driver, skip_release_check, wait_multiplier) except Exception as e: print(e) print(traceback.format_exc()) diff --git a/updates/1.54.0.sh b/updates/1.54.0.sh index dddcf0327..de15a2ae0 100644 --- a/updates/1.54.0.sh +++ b/updates/1.54.0.sh @@ -26,7 +26,7 @@ done echo "done." echo "Updating PHP package signing key..." -apt-get update +apt-get update ||: apt-get install --no-install-recommends -y gnupg2 apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg diff --git a/updates/1.55.0.sh b/updates/1.55.0.sh new file mode 100644 index 000000000..45033e31f --- /dev/null +++ b/updates/1.55.0.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +apt-get update +apt-get install -y --no-install-recommends logrotate