From 38b31b98ec8dfa5b5866712683b42ca8f57e35b7 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 11:35:49 +0100 Subject: [PATCH 01/17] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Updated=20the=20READ?= =?UTF-8?q?ME=20a=20little=20bit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8350366a1..c0960b2a0 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ English | [Traditional Chinese 繁體中文](i18n/README-zh_TW.md) | [Simplified Chinese 简体中文](i18n/README-zh_CN.md) -# NextCloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextCloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextCloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) +# NextcloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextcloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextcloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) -![NC Logo](https://ownyourbits.com/wp-content/uploads/2017/11/ncp-square.png) +![NCP Logo](https://github.com/nextcloud/nextcloudpi/blob/master/ncp-app/img/app.svg) -This is the build code for [NextCloudPi](https://nextcloudpi.com). +This is the build code for [NextcloudPi](https://nextcloudpi.com). -NextCloudPi is a ready to use image for Virtual Machines, Raspberry Pi, Odroid HC1, rock64 and other boards [(⇒Downloads)](https://github.com/nextcloud/nextcloudpi/releases). +NextcloudPi is a ready to use image for Virtual Machines, Raspberry Pi, Odroid HC1, rock64 and other boards [(⇒Downloads)](https://github.com/nextcloud/nextcloudpi/releases). -This code also generates the NextCloudPi [docker image](https://hub.docker.com/r/ownyourbits/nextcloudpi), LXD and VM, and includes an installer for any Debian based system. +This code also generates the NextcloudPi [docker image](https://hub.docker.com/r/ownyourbits/nextcloudpi), LXD and VM, and includes an installer for any Debian based system. Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com) @@ -42,7 +42,7 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co ## Extras * Setup wizard - * NextCloudPi Web Panel + * NextcloudPi Web Panel * Wi-Fi ready * Ram logs * Automatic security updates, activated by default. @@ -98,7 +98,7 @@ docker run -d -p 4443:4443 -p 443:443 -p 80:80 -v ncdata:/data --name nextcloudp ## Run in LXD ``` -lxc import NextCloudPi_LXD_09-29-21.tar.bz +lxc import NextcloudPi_LXD_09-29-21.tar.bz lxc start ncp ``` @@ -118,7 +118,7 @@ build/build-SD-rpi.sh ./build-SD-armbian.sh odroidxu4 # supported board code name ``` -In order to generate the Docker images +In order to generate the Docker images, you'll also need to change the username, repo and tags to match your credentials at Docker Hub. ``` git clone https://github.com/nextcloud/nextcloudpi.git @@ -134,7 +134,7 @@ build/build-docker.sh arm64 build/build-LXD.sh ``` -NextCloudPi can be installed in any architecture running the latest Debian +NextcloudPi can be installed in any architecture running the latest Debian _Note: this assumes a clean Debian install, and there is no rollback method_ @@ -150,4 +150,4 @@ https://hub.docker.com/r/ownyourbits/nextcloudpi ## Contact -You can find us in the [forums](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) and a [Telegram group](https://t.me/NextCloudPi) +You can find us in the [forums](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) and a [Telegram group](https://t.me/NextcloudPi) From 073bbe959f5f87ffbb14b2b67f67c54267987579 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 11:43:20 +0100 Subject: [PATCH 02/17] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Updated=20the=20READ?= =?UTF-8?q?ME=20a=20little=20bit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c0960b2a0..0c5064b1c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ English | [Traditional Chinese 繁體中文](i18n/README-zh_TW.md) | [Simplified # NextcloudPi [![chatroom icon](https://patrolavia.github.io/telegram-badge/chat.png)](https://t.me/NextcloudPi) [![forums icon](https://img.shields.io/badge/help-forums-blue.svg)](https://help.nextcloud.com/c/support/appliances-docker-snappy-vm) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=N8PJHSEQF4G7Y&lc=US&item_name=Own%20Your%20Bits&item_number=NextcloudPi&no_note=1&no_shipping=1¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted) [![blog](https://img.shields.io/badge/follow-blog-orange.svg)](https://ownyourbits.com) - ![NCP Logo](https://github.com/nextcloud/nextcloudpi/blob/master/ncp-app/img/app.svg) This is the build code for [NextcloudPi](https://nextcloudpi.com). @@ -14,6 +13,7 @@ This code also generates the NextcloudPi [docker image](https://hub.docker.com/r Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com) --- + [![VM Integration Tests](https://github.com/nextcloud/nextcloudpi/workflows/VM%20Integration%20Tests/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/vm-tests.yml) [![Docker Integration Tests](https://github.com/nextcloud/nextcloudpi/actions/workflows/build-docker.yml/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/build-docker.yml) @@ -80,13 +80,13 @@ Extras can be activated and configured using the web interface at HTTPS port 444 ![ncp-web](https://user-images.githubusercontent.com/21343324/136853829-f4e99ec0-6307-431f-b4c7-21b2330cae7f.png) -, or from the command line from +Or from the command line using ``` sudo ncp-config ``` -![NCP-config](https://ownyourbits.com/wp-content/uploads/2017/03/ncp-conf-700x456.jpg) +![NCP-config](https://help.nextcloud.com/uploads/default/original/3X/b/3/b3d157022a32296ab54428b14b5df02104a91f18.png) ## Run in docker From 1e2345e00d6f65df796cf6f553842e7ea01b0ae3 Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Sun, 1 Jan 2023 13:07:30 +0100 Subject: [PATCH 03/17] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Fixed=20ncp.sh,=20re?= =?UTF-8?q?moved=20the=20pi=20user=20/sbin/nologin=20shell=20as=20that=20u?= =?UTF-8?q?ser=20no=20longer=20exists=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> --- ncp.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ncp.sh b/ncp.sh index de9d2f7b8..7dde0f283 100644 --- a/ncp.sh +++ b/ncp.sh @@ -1,6 +1,6 @@ #!/bin/bash -# NextCloudPi additions to Raspbian +# NextcloudPi additions to Raspbian # # Copyleft 2017 by Ignacio Nunez Hernanz # GPL licensed (see end of file) * Use at your own risk! @@ -25,12 +25,13 @@ install() $APTINSTALL git dialog whiptail jq file lsb-release mkdir -p "$CONFDIR" "$BINDIR" - # include option in raspi-config (only Raspbian) + # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager + # The raspi-config layout and options have also changed + # https://github.com/RPi-Distro/raspi-config/blob/master/raspi-config test -f /usr/bin/raspi-config && { - sed -i '/Change User Password/i"0 NextCloudPi Configuration" "Configuration of NextCloudPi" \\' /usr/bin/raspi-config - sed -i '/1\\ \*) do_change_pass ;;/i0\\ *) ncp-config ;;' /usr/bin/raspi-config - # Disable raspberry pi default user - usermod pi -s /sbin/nologin + # shellcheck disable=SC1003 + sed -i '/S3 Password/i "S0 NextcloudPi Configuration" "Configuration of NextcloudPi" \\' /usr/bin/raspi-config + sed -i '/S3\\ \*) do_change_pass ;;/i S0\\ *) ncp-config ;;' /usr/bin/raspi-config } # add the ncc shortcut @@ -204,7 +205,7 @@ EOF chmod g+w /var/run/.ncp-latest-version # Install all ncp-apps - ALLOW_UPDATE_SCRIPT=1 bin/ncp-update $BRANCH || exit $? + ALLOW_UPDATE_SCRIPT=1 bin/ncp-update "$BRANCH" || exit $? # LIMIT LOG SIZE grep -q maxsize /etc/logrotate.d/apache2 || sed -i /weekly/amaxsize2M /etc/logrotate.d/apache2 @@ -253,7 +254,7 @@ EOF ## tag image is_docker && local DOCKER_TAG="_docker" is_lxc && local DOCKER_TAG="_lxc" - echo "NextCloudPi${DOCKER_TAG}_$( date "+%m-%d-%y" )" > /usr/local/etc/ncp-baseimage + echo "NextcloudPi${DOCKER_TAG}_$( date "+%m-%d-%y" )" > /usr/local/etc/ncp-baseimage ## SSH hardening if [[ -f /etc/ssh/sshd_config ]]; then From 0cb02a7c6394f471854fc3943a78870827aeea8f Mon Sep 17 00:00:00 2001 From: "Victor-ray, S" <12261439+ZendaiOwl@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:58:02 +0100 Subject: [PATCH 04/17] =?UTF-8?q?=E0=BF=93=E2=9D=AF=20Changed=20NextCloud?= =?UTF-8?q?=20to=20Nextcloud?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Victor-ray, S <12261439+ZendaiOwl@users.noreply.github.com> --- README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0c5064b1c..f0a5c2be5 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,8 @@ Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.co * Automatic NCP updates * Automatic Nextcloud updates * Update notifications - * NextCloud backup and restore - * NextCloud online installation + * Nextcloud backup and restore + * Nextcloud online installation * Format USB drive to BTRFS * BTRFS snapshots * Automatic BTRFS snapshots @@ -104,15 +104,27 @@ lxc start ncp ## How to build -Install git, docker, qemu-user-static, chroot and all the usual building tools. +Install: + +- `git` +- `docker` +- `build-essential` +- `qemu` +- `qemu-user-static` +- `chroot` +- `jq` +- `psmisc` +- `procps` + +and all the usual building tools. ``` git clone https://github.com/nextcloud/nextcloudpi.git cd nextcloudpi -build/build-SD-rpi.sh +./build/build-SD-rpi.sh ``` -, or for an Armbian based board +### Armbian-based board ``` ./build-SD-armbian.sh odroidxu4 # supported board code name @@ -128,16 +140,18 @@ build/build-docker.sh armhf build/build-docker.sh arm64 ``` -, for LXD +### LXD ``` -build/build-LXD.sh +./build/build-LXD.sh ``` NextcloudPi can be installed in any architecture running the latest Debian _Note: this assumes a clean Debian install, and there is no rollback method_ +### Curl install scripts + ``` # curl -sSL https://raw.githubusercontent.com/nextcloud/nextcloudpi/master/install.sh | bash ``` From 73b5000cd66fa0e1a5c46793ccfec9f8f5959512 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:01:26 +0100 Subject: [PATCH 05/17] nc-restore: Fix potential dataloss when restoring backup (fixes #1417) - nc-restore: Keep old data directory if present - nc-restore: Temporarily move and restore user files during nc-restore Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/BACKUPS/nc-backup.sh | 1 + bin/ncp/BACKUPS/nc-restore.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index 240ab4091..b25c3318d 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -114,6 +114,7 @@ tar $compress_arg -cf "$destfile" \ --exclude "nextcloud/data/*/files_versions/*" \ --exclude "nextcloud/data/ncp-update-backups" \ --exclude "nextcloud/data/__groupfolders" \ + --exclude "nextcloud/.data_*" \ -C $basedir nextcloud/ \ || { echo "error generating backup" diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 36a158cc9..2dcd39a3a 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -63,9 +63,31 @@ tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 ## RESTORE FILES echo "restore files..." +[[ -d "$NCDIR/data" ]] && { + DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" + mv -T "$NCDIR/data" "$DATA_BKP_DIR/" +} rm -rf "$NCDIR" mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else +fi + +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else +fi + # update NC database password to this instance sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php From b2b27882f86d6a0729a016241fa250d3ea48ead7 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Tue, 4 Oct 2022 00:26:27 +0200 Subject: [PATCH 06/17] library.sh: Make notify_push setup in set-nc-domain more robust Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- etc/library.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/etc/library.sh b/etc/library.sh index ee94cf3c8..e43a8f456 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -159,7 +159,12 @@ function set-nc-domain() ncc config:system:set trusted_proxies 13 --value="${domain}" ncc config:system:set trusted_proxies 14 --value="$(dig +short "${domain}")" sleep 5 # this seems to be required in the VM for some reason. We get `http2 error: protocol error` after ncp-upgrade-nc - ncc notify_push:setup "${url}/push" + for try in {1..3} + do + echo "Setup notify_push (attempt ${try}/3)" + ncc notify_push:setup "${url}/push" + sleep 5 + done fi } From 59b13d7b953863c995b4964c299efe8a28ec7e75 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:05:28 +0100 Subject: [PATCH 07/17] Fix installation and pipeline errors Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 47 +++++++++++++++++++++++++++++- bin/ncp/CONFIG/nc-datadir.sh | 6 +++- bin/ncp/NETWORKING/SSH.sh | 2 +- build/build-SD-rpi.sh | 2 -- build/docker/Dockerfile | 10 ++++++- etc/library.sh | 33 +++++++++++++++++++++ etc/ncp-config.d/nc-datadir.cfg | 2 +- install.sh | 8 +++++ 8 files changed, 103 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e42579ceb..97e50abf6 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -67,6 +67,41 @@ jobs: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + fetch-depth: 0 + - name: Get previous release + id: get_previous_version + shell: bash + run: | + set -ex + if [[ -n "${{ github.base_ref }}" ]] + then + version="${{ github.base_ref }}" + elif [[ "${{ env.VERSION }}" == "refs/heads/devel" ]] + then + version="latest" + else + if [[ "${{ github.ref_type }}" != "tag" ]] || ! git describe --tags > /dev/null + then + git fetch -fu --tags origin ${{ env.VERSION }}:${{ env.VERSION }} + fi + version="$(git describe --tags || true)" + [[ "$version" =~ .*-.*-.* ]] || { + git checkout HEAD~1 + version="$(git describe --tags || echo latest)" + } + version="${version%-*-*}" + fi + curl "https://hub.docker.com/v2/repositories/ownyourbits/nextcloudpi-${ARCH}/tags/${version}" | grep 'errinfo' > /dev/null && { + echo "Image ownyourbits/nextcloudpi-${ARCH}:${version} does not exist - falling back to :latest" + version="latest" + } + echo "Previous version is '$version'" + echo "PREVIOUS_VERSION=$version" >> "$GITHUB_ENV" + echo "::set-output name=previous_version::${version}" - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -75,7 +110,17 @@ jobs: docker volume create ncdata platform=linux/${ARCH?} [[ "$ARCH" == 'x86' ]] && platform=linux/amd64 - docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + tmpdir=$(mktemp -d) + pushd "$tmpdir" + cat < ./Dockerfile + FROM --platform=$platform ownyourbits/nextcloudpi:${{ steps.get_previous_version.outputs.previous_version }} + + RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf + EOF + docker build . -t ncp-testing -f Dockerfile + popd + rm -r "$tmpdir" + docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing - name: Checkout code uses: actions/checkout@v3 diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index 3a0061d4c..8d55a98f9 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -147,7 +147,11 @@ configure() ( . "${BINDIR?}/SYSTEM/metrics.sh" - reload_metrics_config + reload_metrics_config || { + echo 'WARN: There was an issue reloading ncp metrics. This might not affect your installation, + but keep it in mind if there is an issue with metrics.' + true + } ) echo "The NC data directory has been moved successfully." diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh index 62242e974..227ce47b8 100644 --- a/bin/ncp/NETWORKING/SSH.sh +++ b/bin/ncp/NETWORKING/SSH.sh @@ -39,7 +39,7 @@ configure() echo -e "$PASS\n$CONFIRM" | passwd "$USER" || return 1 # Reenable pi user - usermod "$USER" -s /bin/bash + chsh -s /bin/bash "$USER" # Check for insecure default pi password ( taken from old jessie method ) # TODO Due to Debian bug #1003151 with mkpasswd this feature is not working properly at the moment - https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1837456.html diff --git a/build/build-SD-rpi.sh b/build/build-SD-rpi.sh index d7cc0b709..b7a2d0f19 100755 --- a/build/build-SD-rpi.sh +++ b/build/build-SD-rpi.sh @@ -79,8 +79,6 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ # harden SSH further for Raspbian sed -i 's|^#PermitRootLogin .*|PermitRootLogin no|' /etc/ssh/sshd_config - # default user 'pi' for SSH - # cleanup source etc/library.sh && run_app_unsafe post-inst.sh rm /etc/resolv.conf diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 1f813becb..7d9351a08 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -144,6 +144,7 @@ COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/ COPY build/docker /tmp/ncp-build/build/docker/ COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/ COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg +COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ @@ -160,13 +161,20 @@ touch /.docker-image; \ apt-get update; \ apt-get install --no-install-recommends -y wget ca-certificates; \ -# install nextcloudpi +# install nextcloudpiset_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ source /usr/local/etc/library.sh; \ cd /tmp/ncp-build/; \ install_app ncp.sh; \ mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \ run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \ +echo 'Moving data directory to a more sensible location' \ +mkdir -p /opt/ncdata \ +set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ +run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh \ +set_app_param nc-datadir.sh DATADIR '/data/ncdata' \ +# Remove nc-datadir.cfg as it's disabled in docker +rm /usr/local/etc/nc-datadir.cfg \ sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \ # fix default paths diff --git a/etc/library.sh b/etc/library.sh index e43a8f456..80327d56a 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -246,6 +246,39 @@ find_app_param() jq -r ".params[$p_num].value" < "$cfg_file" } +set_app_param() +{ + local script="${1?}" + local param_id="${2?}" + local param_value="${3?}" + local ncp_app="$(basename "$script" .sh)" + local cfg_file="$CFGDIR/$ncp_app.cfg" + + grep -q '[\\&#;'"'"'`|*?~<>^"()[{}$&[:space:]]' <<< "${param_value}" && { echo "Invalid characters in field ${vars[$i]}"; return 1; } + + cfg="$(cat "$cfg_file")" + + local len="$(jq '.params | length' <<<"$cfg")" + local param_found=false + + for (( i = 0 ; i < len ; i++ )); do + # check for invalid characters + [[ "$(jq -r ".params[$i].id" <<<"$cfg")" == "$param_id" ]] && { + cfg="$(jq ".params[$i].value = \"${param_value}\"" <<<"$cfg")" + param_found=true + } + + done + + [[ "$param_found" == "true" ]] || { + echo "Did not find parameter '${param_id}' in configuration of app '$(basename "$script" .sh)'" + return 1 + } + + echo "$cfg" > "$cfg_file" + +} + # receives a script file, no security checks function run_app_unsafe() { diff --git a/etc/ncp-config.d/nc-datadir.cfg b/etc/ncp-config.d/nc-datadir.cfg index f387c3e3c..7ec1cd724 100644 --- a/etc/ncp-config.d/nc-datadir.cfg +++ b/etc/ncp-config.d/nc-datadir.cfg @@ -9,7 +9,7 @@ { "id": "DATADIR", "name": "Data directory", - "value": "/media/USBdrive/ncdata", + "value": "/opt/ncdata", "suggest": "/media/USBdrive/ncdata", "default": "/var/www/nextcloud/data", "type": "directory" diff --git a/install.sh b/install.sh index b7a459798..ace574556 100644 --- a/install.sh +++ b/install.sh @@ -67,6 +67,14 @@ rm /usr/local/etc/ncp-config.d/nc-nextcloud.cfg # armbian overlay is ro systemctl restart mysqld # TODO this shouldn't be necessary, but somehow it's needed in Debian 9.6. Fixme install_app ncp.sh run_app_unsafe bin/ncp/CONFIG/nc-init.sh +echo 'Moving data directory to a more sensible location' +mkdir -p /opt/ncdata +[[ -f "/usr/local/etc/nc-datadir.cfg" ]] || { + should_rm_datadir_cfg=true + cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/nc-datadir.cfg +} +run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh +[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/nc-datadir.cfg rm /.ncp-image # skip on Armbian / Vagrant / LXD ... From b8ef97f9a475331a177d1a870b2e1abad489e41b Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:07:46 +0100 Subject: [PATCH 08/17] SSH: Add option to enable sudo for ssh user. - Disable root login Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/NETWORKING/SSH.sh | 5 +++++ etc/ncp-config.d/SSH.cfg | 6 ++++++ ncp.sh | 1 + 3 files changed, 12 insertions(+) diff --git a/bin/ncp/NETWORKING/SSH.sh b/bin/ncp/NETWORKING/SSH.sh index 227ce47b8..5d3bbc352 100644 --- a/bin/ncp/NETWORKING/SSH.sh +++ b/bin/ncp/NETWORKING/SSH.sh @@ -41,6 +41,11 @@ configure() # Reenable pi user chsh -s /bin/bash "$USER" + [[ "$SUDO" == "yes" ]] && { + usermod -aG sudo "$USER" + echo "Enabled sudo for $USER" + } + # Check for insecure default pi password ( taken from old jessie method ) # TODO Due to Debian bug #1003151 with mkpasswd this feature is not working properly at the moment - https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1837456.html #local SHADOW SALT HASH diff --git a/etc/ncp-config.d/SSH.cfg b/etc/ncp-config.d/SSH.cfg index 031fb9ee3..388540265 100644 --- a/etc/ncp-config.d/SSH.cfg +++ b/etc/ncp-config.d/SSH.cfg @@ -29,6 +29,12 @@ "name": "Confirm", "value": "1234", "type": "password" + }, + { + "id": "SUDO", + "name": "Enable sudo", + "value": "no", + "type": "bool" } ] } diff --git a/ncp.sh b/ncp.sh index de9d2f7b8..9aab8c11c 100644 --- a/ncp.sh +++ b/ncp.sh @@ -132,6 +132,7 @@ EOF id -u "$WEBADMIN" &>/dev/null || useradd --home-dir /nonexistent "$WEBADMIN" echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN" chsh -s /usr/sbin/nologin "$WEBADMIN" + chsh -s /usr/sbin/nologin root ## NCP LAUNCHER mkdir -p /home/www From 6380ad07abb8ea3fbc9a5d909a3253378343feb2 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:11:52 +0100 Subject: [PATCH 09/17] Installer and pipeline fixes Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 157 +++++++++++++++--------- .github/workflows/build-lxd.yml | 167 ++++++++++++++++++++++---- .github/workflows/build-sd-images.yml | 6 +- .github/workflows/vm-tests.yml | 4 +- bin/ncp-report | 8 +- bin/ncp/BACKUPS/nc-restore.sh | 3 +- bin/ncp/CONFIG/nc-database.sh | 2 +- bin/ncp/CONFIG/nc-datadir.sh | 4 +- build/build-LXD.sh | 5 +- build/build-SD-armbian.sh | 3 +- install.sh | 9 +- 11 files changed, 271 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 97e50abf6..3205c9a4d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -30,14 +30,17 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Checkout code uses: actions/checkout@v3 @@ -66,7 +69,14 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Checkout code uses: actions/checkout@v3 with: @@ -101,9 +111,12 @@ jobs: } echo "Previous version is '$version'" echo "PREVIOUS_VERSION=$version" >> "$GITHUB_ENV" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 + + - name: Enable qemu SUID + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes - name: Start ncp container run: | @@ -117,10 +130,10 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker build . -t ncp-testing -f Dockerfile + docker buildx build --platform "$platform" . -t ncp-testing -f Dockerfile popd rm -r "$tmpdir" - docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing + docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost - name: Checkout code uses: actions/checkout@v3 @@ -137,10 +150,10 @@ jobs: - name: Activate container working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) [[ "${ARCH?}" == "x86" ]] || { sleep 30; cmd+=(--timeout 300); } @@ -148,10 +161,13 @@ jobs: for attempt in {1..3} do echo ":: Activation Tests (attempt $attempt/3) ::" - "${cmd[@]}" || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Activation test (attempt $attempt/3) failed!" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + cat /proc/sys/fs/binfmt_misc/qemu-aarch64 |& awk "{ print \"${LOG_DIAG} (qemu-aarch64) \" \$0 }" || true + cat /proc/sys/fs/binfmt_misc/qemu-arm |& awk "{ print \"${LOG_DIAG} (qemu-arm) \" \$0 }" || true sleep 12 continue } @@ -159,21 +175,24 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Activation test failed in all attempts!" + echo -e "${LOG_CICD} Activation test failed in all attempts!" + echo "Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo "done." exit 1 } - echo "CI:: Activation test successful" + echo -e "${LOG_CICD} Activation test successful" - name: Start new container run: | docker stop nextcloudpi - docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing + docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost - name: Integration Tests working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & [[ "${ARCH?}" == "x86" ]] || sleep 30 sleep 20 @@ -182,8 +201,9 @@ jobs: for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || { - echo "CI:: System test (attempt $attempt) failed!" + python system_tests.py --no-ping --non-interactive || awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 12 continue } @@ -191,18 +211,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: System test failed in all attempts!" + echo -e "${LOG_CICD} System test failed in all attempts!" exit 1 } - echo "CI:: System test successful" + echo -e "${LOG_CICD} System test successful" success=false for attempt in {1..3} do echo ":: Nextcloud Tests (attempt $attempt/3) ::" - python nextcloud_tests.py --no-gui localhost 8443 4443 || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!" sleep 12 continue } @@ -210,12 +231,21 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo -e "${LOG_CICD} done." exit 1 } - echo "CI:: Nextcloud test successful" + echo -e "${LOG_CICD} Nextcloud test successful" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" + - name: "Upload error report" + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-docker-${{ env.ARCH }}-update-test-error-report + path: tests/error-report.txt test: needs: @@ -224,16 +254,21 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARCH: "${{ inputs.arch || 'x86' }}" + LOG_DCKR: "\\033[1;34mDCKR::\\033[0m" + LOG_NCP: "\\033[1;36m~NCP::\\033[0m" + LOG_CICD: "\\033[1;35mCICD::\\033[0m" + LOG_TEST: "\\033[1;33mTEST::\\033[0m" + LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Login to docker run: | echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin - name: Start ncp container run: | - docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing + docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing localhost - name: Checkout code uses: actions/checkout@v3 @@ -250,10 +285,10 @@ jobs: - name: Integration Tests working-directory: ./tests run: | - docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' - echo 'CONTAINER:: ==========' - docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & - docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + docker logs nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" + echo -e "${LOG_DCKR} ==========" + docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & cmd=(python activation_tests.py --no-gui localhost 8443 4443) [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) @@ -261,10 +296,11 @@ jobs: for attempt in {1..3} do echo ":: Activation Tests (attempt $attempt/3) ::" - "${cmd[@]}" || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Activation test (attempt $attempt/3) failed!" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true sleep 12 continue } @@ -272,17 +308,18 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Activation test failed in all attempts!" + echo -e "${LOG_CICD} Activation test failed in all attempts!" exit 1 } - echo "CI:: Activation test successful" + echo -e "${LOG_CICD} Activation test successful" success=false for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || { - echo "CI:: System test (attempt $attempt) failed!" + python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 5 continue } @@ -290,18 +327,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: System test failed in all attempts!" + echo -e "${LOG_CICD} System test failed in all attempts!" exit 1 } - echo "CI:: System test successful" + echo -e "${LOG_CICD} System test successful" success=false for attempt in {1..3} do echo ":: Nextcloud Tests (attempt $attempt/3) ::" - python nextcloud_tests.py --no-gui localhost 8443 4443 || { - tail -n 20 geckodriver.log >&2 || true - echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" + [[ ${PIPESTATUS[0]} -eq 0 ]] || { + tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + echo -e "${LOG_CICD} Nextcloud test (attempt $attempt/3) failed!" sleep 5 continue } @@ -309,10 +347,19 @@ jobs: break done [[ "$success" == "true" ]] || { - echo "CI:: Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" + echo -e "${LOG_CICD} Creating error report..." + docker exec nextcloudpi bash /usr/local/bin/ncp-report > error-report.txt + echo -e "${LOG_CICD} done." exit 1 } - echo "CI:: Nextcloud test successful" + echo -e "${LOG_CICD} Nextcloud test successful" - docker exec nextcloudpi bash /usr/local/bin/ncp-diag + docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + - name: "Upload error report" + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-docker-${{ env.ARCH }}-install-test-error-report + path: tests/error-report.txt diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 223a8ffbe..ef1ccab39 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -19,8 +19,8 @@ on: - "devel" jobs: - build: - runs-on: ubuntu-latest + build-current: + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -35,6 +35,28 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + +# - name: Fix LXD +# run: | +# ip address +# ip route list +# lxc profile create network +# cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -53,7 +75,7 @@ jobs: if-no-files-found: error build-previous: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -61,6 +83,7 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" ARTIFACT_NAME: "${{ github.run_id }}-lxd-image-previous" + LXD_EXTRA_PROFILE: network steps: - name: Checkout code uses: actions/checkout@v3 @@ -93,10 +116,21 @@ jobs: echo "Previous version is '$version'" git checkout "$version" echo "VERSION=$version" >> "$GITHUB_ENV" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -117,7 +151,7 @@ jobs: update-previous: needs: - build-previous - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 outputs: artifact_name: "${{ env.ARTIFACT_NAME }}" artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}" @@ -128,6 +162,17 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat <> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 with: @@ -193,23 +238,89 @@ jobs: path: "output/${{ steps.pack-lxd.outputs.artifact_file }}" if-no-files-found: error - test: + test-fresh-install: + needs: + - build-current + runs-on: ubuntu-20.04 + env: + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} + ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + - uses: whywaita/setup-lxd@v1 + with: + lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sleep 30 + ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Test LXD Image + working-directory: ./tests + run: | + lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Activation test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + python system_tests.py --non-interactive || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + lxc stop ncp + + test-update: needs: - - build - update-previous - strategy: - matrix: - build: - - source: install - artifact_name: ${{ needs.build.outputs.artifact_name }} - artifact_file: ${{ needs.build.outputs.artifact_file }} - - source: update - artifact_name: ${{ needs.update-previous.outputs.artifact_name }} - artifact_file: ${{ needs.update-previous.outputs.artifact_file }} - fail-fast: false - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + ARTIFACT_NAME: ${{ needs.update-previous.outputs.artifact_name }} + ARTIFACT_FILE: ${{ needs.update-previous.outputs.artifact_file }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -218,6 +329,16 @@ jobs: - uses: whywaita/setup-lxd@v1 with: lxd_version: latest/stable + - name: Fix LXD + run: | + lxc profile create network + cat </dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index 08af6d47c..168dc4e67 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -68,7 +68,7 @@ jobs: artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Build Armbian (2nd attempt)" if: ${{ inputs.board_id != 'raspberrypi' && steps.build-armbian.outcome == 'failure' }} @@ -87,7 +87,7 @@ jobs: artifacts=("armbian/output/images/Armbian"*.img) mkdir -p output mv "${artifacts[0]}" "output/$IMG" - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: "Upload Armbian logs" if: ${{ inputs.board_id != 'raspberrypi' && failure() }} @@ -116,7 +116,7 @@ jobs: echo "Retrying ($i out of 10)" done - echo "::set-output name=artifact_file::${IMG}" + echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV - name: upload image to artifact store uses: actions/upload-artifact@v3 diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index f2ff3cd39..743ae5b4d 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -81,7 +81,7 @@ jobs: version="${version%-*-*}" fi echo "Previous version is '$version'" - echo "::set-output name=previous_version::${version}" + echo "previous_version=${version}" >> $GITHUB_OUTPUT - run: | set -x mkdir -p ./.ssh @@ -387,7 +387,7 @@ jobs: steps: - name: Delete old snapshots run: | - for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -20) + for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -16) do echo "Deleting snapshot '$snapshot'..." hcloud image delete "$snapshot" diff --git a/bin/ncp-report b/bin/ncp-report index e598b2439..7ca5af9c1 100755 --- a/bin/ncp-report +++ b/bin/ncp-report @@ -45,19 +45,19 @@ close_summary ## open_summary "Nextcloud configuration" -sudo -u www-data php /var/www/nextcloud/occ config:list system +sudo -u www-data php /var/www/nextcloud/occ config:list system 2>&1 || true close_summary ## open_summary "HTTPd logs" -tail -20 /var/log/apache2/error.log +tail -20 /var/log/apache2/error.log 2>&1 || true close_summary ## open_summary "Database logs" -tail -20 /var/log/mysql/*.log +tail -20 /var/log/mysql/*.log 2>&1 || true close_summary ## @@ -66,7 +66,7 @@ DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )" open_summary "Nextcloud logs" -tail -20 "$DATADIR"/nextcloud.log +tail -20 "$DATADIR"/nextcloud.log 2>&1 || true close_summary # License diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 2dcd39a3a..97d88e6b0 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -31,7 +31,7 @@ DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath [[ "$DIR" =~ "$NCDIR" ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } -grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems" >&2; exit 1; } +grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } @@ -77,6 +77,7 @@ then [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" mv -T "$DATA_BKP_DIR" "$target" else + echo "Not overwriting data directory" fi if [[ -n "$DATA_BKP_DIR" ]] diff --git a/bin/ncp/CONFIG/nc-database.sh b/bin/ncp/CONFIG/nc-database.sh index 54b10d358..013cdf44f 100644 --- a/bin/ncp/CONFIG/nc-database.sh +++ b/bin/ncp/CONFIG/nc-database.sh @@ -37,7 +37,7 @@ configure() local BASEDIR=$( dirname "$DBDIR" ) mkdir -p "$BASEDIR" - grep -q -e ext -e btrfs <( stat -fc%T "$BASEDIR" ) || { echo -e "Only ext/btrfs filesystems can hold the data directory"; return 1; } + grep -q -e ext -e btrfs <( stat -fc%T "$BASEDIR" ) || { echo -e "Only ext/btrfs filesystems can hold the data directory (found '$(stat -fc%T "${BASEDIR}")"; return 1; } sudo -u mysql test -x "$BASEDIR" || { echo -e "ERROR: the user mysql does not have access permissions over $BASEDIR"; return 1; } diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index 8d55a98f9..df7869b11 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -79,8 +79,8 @@ configure() ENCDIR="${BASEDIR}"/ncdata_enc # checks - grep -q -e ext -e btrfs <( stat -fc%T "${BASEDIR}" ) || { - echo -e "Only ext/btrfs filesystems can hold the data directory" + [[ "$DISABLE_FS_CHECK" == 1 ]] || grep -q -e ext -e btrfs <( stat -fc%T "${BASEDIR}" ) || { + echo -e "Only ext/btrfs filesystems can hold the data directory (found '$(stat -fc%T "${BASEDIR}")')" return 1 } diff --git a/build/build-LXD.sh b/build/build-LXD.sh index c695e769e..e34402e10 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -32,7 +32,10 @@ prepare_dirs # tmp cache output ## BUILD NCP lxc delete -f ncp 2>/dev/null || true -systemd-run --user --scope -p "Delegate=yes" lxc launch -q images:debian/bullseye ncp +LXC_LAUNCH=(lxc launch -p default) +[[ -n "$LXD_EXTRA_PROFILE" ]] && LXC_LAUNCH+=(-p "$LXD_EXTRA_PROFILE") +LXC_LAUNCH+=(-q 'images:debian/bullseye' ncp) +systemd-run --user --scope -p "Delegate=yes" "${LXC_LAUNCH[@]}" lxc config device add ncp buildcode disk source="$(pwd)" path=/build lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index fddc30c5a..1a955c56e 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -69,7 +69,8 @@ EXTRA_CONF=build/armbian/"config-$BOARD".conf rm -rf armbian/output/images mkdir -p armbian/userpatches sed -e '/docker.*run/s/-it//' armbian/config/templates/config-docker.conf > armbian/userpatches/config-docker.conf -docker pull "ghcr.io/armbian/build:$(cut -d"." -f1-2 < armbian/VERSION)-$(dpkg --print-architecture)" +docker pull "ghcr.io/armbian/build:$(cut -d"." -f1-2 < armbian/VERSION)-$(dpkg --print-architecture)" \ + || docker pull "ghcr.io/armbian/build:latest-$(dpkg --print-architecture)" armbian/compile.sh docker ncp rm "$CONF" diff --git a/install.sh b/install.sh index ace574556..31044e2bb 100644 --- a/install.sh +++ b/install.sh @@ -68,13 +68,14 @@ systemctl restart mysqld # TODO this shouldn't be necessary, but somehow it's ne install_app ncp.sh run_app_unsafe bin/ncp/CONFIG/nc-init.sh echo 'Moving data directory to a more sensible location' +df -h mkdir -p /opt/ncdata -[[ -f "/usr/local/etc/nc-datadir.cfg" ]] || { +[[ -f "/usr/local/etc/ncp-config.d/nc-datadir.cfg" ]] || { should_rm_datadir_cfg=true - cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/nc-datadir.cfg + cp etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg } -run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh -[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/nc-datadir.cfg +DISABLE_FS_CHECK=1 NCPCFG="/usr/local/etc/ncp.cfg" run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh +[[ -z "$should_rm_datadir_cfg" ]] || rm /usr/local/etc/ncp-config.d/nc-datadir.cfg rm /.ncp-image # skip on Armbian / Vagrant / LXD ... From 25e51f6fbcc48e3641548d8a61d79f1e365b4d5d Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 14:48:14 +0100 Subject: [PATCH 10/17] Move ncp-backup and ncp-restore to separate files (instead relying on cat during installation) Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-backup | 104 ++++++++++++++++++ bin/ncp-restore | 184 +++++++++++++++++++++++++++++++ bin/ncp/BACKUPS/nc-backup.sh | 110 +------------------ bin/ncp/BACKUPS/nc-restore.sh | 198 +--------------------------------- 4 files changed, 292 insertions(+), 304 deletions(-) create mode 100755 bin/ncp-backup create mode 100755 bin/ncp-restore diff --git a/bin/ncp-backup b/bin/ncp-backup new file mode 100755 index 000000000..f7e22a431 --- /dev/null +++ b/bin/ncp-backup @@ -0,0 +1,104 @@ +#!/bin/bash +set -eE +source /usr/local/etc/library.sh + +if [ "$1" = "--help" ]; then + echo "$0 " + echo " and are 'yes' or 'no'" + echo " integer >= 1 or 0 for no limit" + exit 0 +fi + +destdir="${1:-/media/USBdrive/ncp-backups}" +includedata="${2:-no}" +compress="${3:-no}" +backuplimit="${4:-0}" + +destfile="$destdir"/nextcloud-bkp_$( date +"%Y%m%d_%s" ).tar +dbbackup=nextcloud-sqlbkp_$( date +"%Y%m%d" ).bak +[[ -f /.docker-image ]] && basedir=/data || basedir=/var/www + +[[ "$compress" == "yes" ]] && destfile="$destfile".gz + +datadir=$( get_nc_config_value datadirectory ) || { + echo "Error reading data directory. Is NextCloud running and configured?"; + exit 1; +} + +cleanup(){ local ret=$?; rm -f "${dbbackup}" ; restore_maintenance_mode; exit $ret; } +fail() { local ret=$?; echo "Abort..." ; rm -f "${dbbackup}" "${destfile}"; restore_maintenance_mode; exit $ret; } +trap cleanup EXIT +trap fail INT TERM HUP ERR + +echo "check free space..." # allow at least ~500 extra MiB +mkdir -p "$destdir" +nsize=$(du -sb "$basedir/nextcloud" | awk '{ print $1 }') +[[ -d "$basedir/nextcloud"/data ]] && \ + dsize=$(du -sb "$basedir/nextcloud"/data | awk '{ print $1 }') +margin=$((500*1024*1024)) # safety margin for database and some extra +if [[ "$includedata" == "yes" ]]; then + datasize=$(du -sb "$datadir" | awk '{ print $1 }') + size=$((nsize - dsize + margin + datasize)) +else + size=$((nsize - dsize + margin)) +fi +free=$( df -B1 "$destdir" | tail -1 | awk '{ print $4 }' ) + +[ $size -ge $free ] && { + echo "free space check failed. Need $size Bytes"; + exit 1; +} + +# delete older backups +[[ $backuplimit != 0 ]] && { + numbkps=$( ls "$destdir"/nextcloud-bkp_* 2>/dev/null | wc -l ) + [[ $numbkps -ge $backuplimit ]] && \ + ls -t $destdir/nextcloud-bkp_* | tail -$(( numbkps - backuplimit + 1 )) | while read -r f; do + echo "clean up old backup $f" + rm "$f" + done +} + +# database +save_maintenance_mode +cd "$basedir" || exit 1 +echo "backup database..." +mysqldump -u root --single-transaction nextcloud > "$dbbackup" + +# files +echo "backup files..." +[[ "$includedata" == "yes" ]] && data="$(basename "$datadir")" +[[ "$compress" == "yes" ]] && compress_arg="-I pigz" +mkdir -p "$destdir" +tar $compress_arg -cf "$destfile" \ +\ + "$dbbackup" \ +\ + --exclude "$data/.opcache" \ + --exclude "$data/{access,error,nextcloud}.log" \ + --exclude "$data/access.log" \ + --exclude "$data/ncp-update-backups" \ + -C "$(dirname "$datadir"/)" $data \ +\ + --exclude "nextcloud/data/*/files/*" \ + --exclude "nextcloud/data/.opcache" \ + --exclude "nextcloud/data/{access,error,nextcloud}.log" \ + --exclude "nextcloud/data/access.log" \ + --exclude "nextcloud/data/appdata_*/preview/*" \ + --exclude "nextcloud/data/*/uploads/*" \ + --exclude "nextcloud/data/*/cache/*" \ + --exclude "nextcloud/data/*/files_trashbin/*" \ + --exclude "nextcloud/data/*/files_versions/*" \ + --exclude "nextcloud/data/ncp-update-backups" \ + --exclude "nextcloud/data/__groupfolders" \ + --exclude "nextcloud/.data_*" \ + -C $basedir nextcloud/ \ + || { + echo "error generating backup" + exit 1 + } +rm "$dbbackup" +chmod 640 "$destfile" +chown :www-data "$destfile" + +echo "backup $destfile generated" \ No newline at end of file diff --git a/bin/ncp-restore b/bin/ncp-restore new file mode 100755 index 000000000..9b63df43d --- /dev/null +++ b/bin/ncp-restore @@ -0,0 +1,184 @@ +#!/bin/bash +set -eE + +BACKUPFILE="$1" + +DBADMIN=ncadmin +DBPASSWD="$( grep password /root/.my.cnf | sed 's|password=||' )" + +source /usr/local/etc/library.sh # sets PHPVER + +DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath + +[[ -f /.docker-image ]] && NCDIR=/data/nextcloud || NCDIR=/var/www/nextcloud + +[[ $# -eq 0 ]] && { echo "missing first argument" ; exit 1; } +[[ -f "$BACKUPFILE" ]] || { echo "$BACKUPFILE not found" ; exit 1; } +[[ "$DIR" =~ "$NCDIR".* ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } + +TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } +grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } + +TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath +cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } +trap cleanup INT TERM HUP ERR EXIT +rm -rf "$TMPDIR" && mkdir -p "$TMPDIR" + +[[ "$BACKUPFILE" =~ .*".tar.gz" ]] && compress_arg="-I pigz" + +# CHECK FREE SPACE IN $TMPDIR +echo "check free space..." # allow at least ~100 extra MiB +extractedsize=$(tar $compress_arg -tvf "$BACKUPFILE" | awk '{s+=$3} END{printf "%.0f", (s/1024)}') # Size of extracted files in "KB" +size=$(($extractedsize + 100*1024)) +free=$( df "$TMPDIR" | tail -1 | awk '{ print $4 }' ) +[ $size -ge $free ] && { + echo "free space check failed. Need $size KB in $TMPDIR"; + exit 1; +} + +# EXTRACT FILES +echo "extracting backup file $BACKUPFILE..." +tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 + +## SANITY CHECKS +[[ -d "$TMPDIR"/nextcloud ]] && [[ -f "$( ls "$TMPDIR"/nextcloud-sqlbkp_*.bak 2>/dev/null )" ]] || { + echo "invalid backup file. Abort" + exit 1 +} + +## RESTORE FILES + +echo "restore files..." +[[ -d "$NCDIR/data" ]] && { + DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" + mv -T "$NCDIR/data" "$DATA_BKP_DIR/" +} +rm -rf "$NCDIR" +mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } + +if [[ -n "$DATA_BKP_DIR" ]] +then + echo "Restoring $NCDIR/data..." + target="$NCDIR/data" + [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" + mv -T "$DATA_BKP_DIR" "$target" +else + echo "Dataless backup detected - not restoring data" +fi + +# update NC database password to this instance +sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php + +# update redis credentials +REDISPASS="$( grep "^requirepass" /etc/redis/redis.conf | cut -f2 -d' ' )" +[[ "$REDISPASS" != "" ]] && \ + sed -i "s|'password'.*|'password' => '$REDISPASS',|" /var/www/nextcloud/config/config.php +service redis-server restart + +## RE-CREATE DATABASE TABLE + +echo "restore database..." +mysql -u root </dev/null && { + rmdir "$DATADIR" || exit 1 + btrfs subvolume create "$DATADIR" || exit 1 + } + chown www-data: "$DATADIR" + TMPDATA="$TMPDIR/$( basename "$DATADIR" )" + mv "$TMPDATA"/* "$TMPDATA"/.[!.]* "$DATADIR" || exit 1 + rmdir "$TMPDATA" || exit 1 + + ncc maintenance:mode --off + +### INCLUDEDATA=no situation + +else + echo "No datadir found in backup" + + [[ -e "$DATADIR" ]] || { + echo "${DATADIR} not found. Resetting to ${NCDIR}/data" + DATADIR="$NCDIR"/data + mkdir -p "${DATADIR}" + touch "${DATADIR}"/.ocdata + chown -R www-data: "${DATADIR}" + } + + ncc maintenance:mode --off + ncc files:scan --all + + # cache needs to be cleaned as of NC 12 + NEED_RESTART=1 +fi + +sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php + +# Just in case we moved the opcache dir +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" + +# tmp upload dir +mkdir -p "$DATADIR/tmp" +chown www-data: "$DATADIR/tmp" +ncc config:system:set tempdirectory --value "$DATADIR/tmp" +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/cli/php.ini +sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini +sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini + +# logs +ncc config:system:set logfile --value="$DATADIR/nextcloud.log" + +# update fail2ban logpath +[[ -f /etc/fail2ban/jail.conf ]] && { + sed -i "s|logpath =.*|logpath = $DATADIR/nextcloud.log|" /etc/fail2ban/jail.conf + pgrep fail2ban &>/dev/null && service fail2ban restart +} + +# refresh nextcloud trusted domains +bash /usr/local/bin/nextcloud-domain.sh + +# update the systems data-fingerprint +ncc maintenance:data-fingerprint + +# refresh thumbnails +ncc files:scan-app-data + +# restart PHP if needed +[[ "$NEED_RESTART" == "1" ]] && { + clear_opcache + bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & +} \ No newline at end of file diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index b25c3318d..1b08ba745 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -19,114 +19,8 @@ install() apt-get update apt-get install -y --no-install-recommends pigz - cat > /usr/local/bin/ncp-backup <<'EOF' -#!/bin/bash -set -eE -source /usr/local/etc/library.sh - -if [ "$1" = "--help" ]; then - echo "$0 " - echo " and are 'yes' or 'no'" - echo " integer >= 1 or 0 for no limit" - exit 0 -fi - -destdir="${1:-/media/USBdrive/ncp-backups}" -includedata="${2:-no}" -compress="${3:-no}" -backuplimit="${4:-0}" - -destfile="$destdir"/nextcloud-bkp_$( date +"%Y%m%d_%s" ).tar -dbbackup=nextcloud-sqlbkp_$( date +"%Y%m%d" ).bak -occ="sudo -u www-data php /var/www/nextcloud/occ" -[[ -f /.docker-image ]] && basedir=/data || basedir=/var/www - -[[ "$compress" == "yes" ]] && destfile="$destfile".gz - -datadir=$( get_nc_config_value datadirectory ) || { - echo "Error reading data directory. Is NextCloud running and configured?"; - exit 1; -} - -cleanup(){ local ret=$?; rm -f "${dbbackup}" ; restore_maintenance_mode; exit $ret; } -fail() { local ret=$?; echo "Abort..." ; rm -f "${dbbackup}" "${destfile}"; restore_maintenance_mode; exit $ret; } -trap cleanup EXIT -trap fail INT TERM HUP ERR - -echo "check free space..." # allow at least ~500 extra MiB -mkdir -p "$destdir" -nsize=$(du -sb "$basedir/nextcloud" | awk '{ print $1 }') -[[ -d "$basedir/nextcloud"/data ]] && \ - dsize=$(du -sb "$basedir/nextcloud"/data | awk '{ print $1 }') -margin=$((500*1024*1024)) # safety margin for database and some extra -if [[ "$includedata" == "yes" ]]; then - datasize=$(du -sb "$datadir" | awk '{ print $1 }') - size=$((nsize - dsize + margin + datasize)) -else - size=$((nsize - dsize + margin)) -fi -free=$( df -B1 "$destdir" | tail -1 | awk '{ print $4 }' ) - -[ $size -ge $free ] && { - echo "free space check failed. Need $size Bytes"; - exit 1; -} - -# delete older backups -[[ $backuplimit != 0 ]] && { - numbkps=$( ls "$destdir"/nextcloud-bkp_* 2>/dev/null | wc -l ) - [[ $numbkps -ge $backuplimit ]] && \ - ls -t $destdir/nextcloud-bkp_* | tail -$(( numbkps - backuplimit + 1 )) | while read -r f; do - echo "clean up old backup $f" - rm "$f" - done -} - -# database -save_maintenance_mode -cd "$basedir" || exit 1 -echo "backup database..." -mysqldump -u root --single-transaction nextcloud > "$dbbackup" - -# files -echo "backup files..." -[[ "$includedata" == "yes" ]] && data="$(basename "$datadir")" -[[ "$compress" == "yes" ]] && compress_arg="-I pigz" -mkdir -p "$destdir" -tar $compress_arg -cf "$destfile" \ -\ - "$dbbackup" \ -\ - --exclude "$data/.opcache" \ - --exclude "$data/{access,error,nextcloud}.log" \ - --exclude "$data/access.log" \ - --exclude "$data/ncp-update-backups" \ - -C "$(dirname "$datadir"/)" $data \ -\ - --exclude "nextcloud/data/*/files/*" \ - --exclude "nextcloud/data/.opcache" \ - --exclude "nextcloud/data/{access,error,nextcloud}.log" \ - --exclude "nextcloud/data/access.log" \ - --exclude "nextcloud/data/appdata_*/preview/*" \ - --exclude "nextcloud/data/*/uploads/*" \ - --exclude "nextcloud/data/*/cache/*" \ - --exclude "nextcloud/data/*/files_trashbin/*" \ - --exclude "nextcloud/data/*/files_versions/*" \ - --exclude "nextcloud/data/ncp-update-backups" \ - --exclude "nextcloud/data/__groupfolders" \ - --exclude "nextcloud/.data_*" \ - -C $basedir nextcloud/ \ - || { - echo "error generating backup" - exit 1 - } -rm "$dbbackup" -chmod 640 "$destfile" -chown :www-data "$destfile" - -echo "backup $destfile generated" -EOF - chmod +x /usr/local/bin/ncp-backup + ln -s /usr/local/bin/ncp-backup /usr/local/bin/ncp/ncp-backup + chmod +x /usr/local/bin/ncp/ncp-backup } configure() diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 97d88e6b0..322fab9d6 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -11,202 +11,8 @@ install() { - cat > /usr/local/bin/ncp-restore <<'EOF' -#!/bin/bash -set -eE - -BACKUPFILE="$1" - -DBADMIN=ncadmin -DBPASSWD="$( grep password /root/.my.cnf | sed 's|password=||' )" - -source /usr/local/etc/library.sh # sets PHPVER - -DIR="$( cd "$( dirname "$BACKUPFILE" )" &>/dev/null && pwd )" #abspath - -[[ -f /.docker-image ]] && NCDIR=/data/nextcloud || NCDIR=/var/www/nextcloud - -[[ $# -eq 0 ]] && { echo "missing first argument" ; exit 1; } -[[ -f "$BACKUPFILE" ]] || { echo "$BACKUPFILE not found" ; exit 1; } -[[ "$DIR" =~ "$NCDIR" ]] && { echo "Refusing to restore from $NCDIR"; exit 1; } - -TMPDIR="$( mktemp -d "$( dirname "$BACKUPFILE" )"/ncp-restore.XXXXXX )" || { echo "Failed to create temp dir" >&2; exit 1; } -grep -q -e ext -e btrfs -e zfs <( stat -fc%T "$TMPDIR" ) || { echo "Can only restore from ext/btrfs/zfs filesystems (found '$(stat -fc%T "${TMPDIR}")" >&2; exit 1; } - -TMPDIR="$( cd "$TMPDIR" &>/dev/null && pwd )" || { echo "$TMPDIR not found"; exit 1; } #abspath -cleanup(){ local RET=$?; echo "Cleanup..."; rm -rf "${TMPDIR}"; trap "" EXIT; exit $RET; } -trap cleanup INT TERM HUP ERR EXIT -rm -rf "$TMPDIR" && mkdir -p "$TMPDIR" - -[[ "$BACKUPFILE" =~ ".tar.gz" ]] && compress_arg="-I pigz" - -# CHECK FREE SPACE IN $TMPDIR -echo "check free space..." # allow at least ~100 extra MiB -extractedsize=$(tar $compress_arg -tvf "$BACKUPFILE" | awk '{s+=$3} END{printf "%.0f", (s/1024)}') # Size of extracted files in "KB" -size=$(($extractedsize + 100*1024)) -free=$( df "$TMPDIR" | tail -1 | awk '{ print $4 }' ) -[ $size -ge $free ] && { - echo "free space check failed. Need $size KB in $TMPDIR"; - exit 1; -} - -# EXTRACT FILES -echo "extracting backup file $BACKUPFILE..." -tar $compress_arg -xf "$BACKUPFILE" -C "$TMPDIR" || exit 1 - -## SANITY CHECKS -[[ -d "$TMPDIR"/nextcloud ]] && [[ -f "$( ls "$TMPDIR"/nextcloud-sqlbkp_*.bak 2>/dev/null )" ]] || { - echo "invalid backup file. Abort" - exit 1 -} - -## RESTORE FILES - -echo "restore files..." -[[ -d "$NCDIR/data" ]] && { - DATA_BKP_DIR="$(mktemp -d -p "$NCDIR/.." nc-data-XXXXXX)" - mv -T "$NCDIR/data" "$DATA_BKP_DIR/" -} -rm -rf "$NCDIR" -mv -T "$TMPDIR"/nextcloud "$NCDIR" || { echo "Error restoring base files"; exit 1; } - -if [[ -n "$DATA_BKP_DIR" ]] -then - echo "Restoring $NCDIR/data..." - target="$NCDIR/data" - [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" - mv -T "$DATA_BKP_DIR" "$target" -else - echo "Not overwriting data directory" -fi - -if [[ -n "$DATA_BKP_DIR" ]] -then - echo "Restoring $NCDIR/data..." - target="$NCDIR/data" - [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" - mv -T "$DATA_BKP_DIR" "$target" -else -fi - -# update NC database password to this instance -sed -i "s|'dbpassword' =>.*|'dbpassword' => '$DBPASSWD',|" /var/www/nextcloud/config/config.php - -# update redis credentials -REDISPASS="$( grep "^requirepass" /etc/redis/redis.conf | cut -f2 -d' ' )" -[[ "$REDISPASS" != "" ]] && \ - sed -i "s|'password'.*|'password' => '$REDISPASS',|" /var/www/nextcloud/config/config.php -service redis-server restart - -## RE-CREATE DATABASE TABLE - -echo "restore database..." -mysql -u root </dev/null && { - rmdir "$DATADIR" || exit 1 - btrfs subvolume create "$DATADIR" || exit 1 - } - chown www-data: "$DATADIR" - TMPDATA="$TMPDIR/$( basename "$DATADIR" )" - mv "$TMPDATA"/* "$TMPDATA"/.[!.]* "$DATADIR" || exit 1 - rmdir "$TMPDATA" || exit 1 - - ncc maintenance:mode --off - -### INCLUDEDATA=no situation - -else - echo "No datadir found in backup" - - [[ -e "$DATADIR" ]] || { - echo "${DATADIR} not found. Resetting to ${NCDIR}/data" - DATADIR="$NCDIR"/data - mkdir -p "${DATADIR}" - touch "${DATADIR}"/.ocdata - chown -R www-data: "${DATADIR}" - } - - ncc maintenance:mode --off - ncc files:scan --all - - # cache needs to be cleaned as of NC 12 - NEED_RESTART=1 -fi - -sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php - -# Just in case we moved the opcache dir -install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" - -# tmp upload dir -mkdir -p "$DATADIR/tmp" -chown www-data: "$DATADIR/tmp" -ncc config:system:set tempdirectory --value "$DATADIR/tmp" -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/cli/php.ini -sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini -sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHPVER}/fpm/php.ini - -# logs -ncc config:system:set logfile --value="$DATADIR/nextcloud.log" - -# update fail2ban logpath -[[ -f /etc/fail2ban/jail.conf ]] && { - sed -i "s|logpath =.*|logpath = $DATADIR/nextcloud.log|" /etc/fail2ban/jail.conf - pgrep fail2ban &>/dev/null && service fail2ban restart -} - -# refresh nextcloud trusted domains -bash /usr/local/bin/nextcloud-domain.sh - -# update the systems data-fingerprint -ncc maintenance:data-fingerprint - -# refresh thumbnails -ncc files:scan-app-data - -# restart PHP if needed -[[ "$NEED_RESTART" == "1" ]] && { - clear_opcache - bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & -} -EOF - chmod +x /usr/local/bin/ncp-restore + ln -s /usr/local/bin/ncp-restore /usr/local/bin/ncp/ncp-restore + chmod +x /usr/local/bin/ncp/ncp-restore } configure() From 14039536ede0f4cb2a97a1536d8bd87fb5daa5cf Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:12:59 +0100 Subject: [PATCH 11/17] Add basic integration test for nc-backup Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- tests/NcpRobotLib.py | 178 +++++++++++++++++++++++++++++++++++++ tests/nc_backup_test.robot | 23 +++++ tests/requirements.txt | 2 + 3 files changed, 203 insertions(+) create mode 100644 tests/NcpRobotLib.py create mode 100644 tests/nc_backup_test.robot create mode 100644 tests/requirements.txt diff --git a/tests/NcpRobotLib.py b/tests/NcpRobotLib.py new file mode 100644 index 000000000..cc44939ba --- /dev/null +++ b/tests/NcpRobotLib.py @@ -0,0 +1,178 @@ +import json +import os +import re +import subprocess +from subprocess import PIPE +import random +import string +import time +from typing import Dict +from robot.api import FatalError, Error, logger +import tempfile + + +class NcpRobotLib: + ROBOT_LIBRARY_SCOPE = "SUITE" + + def __init__(self, http_port=80, https_port=443, webui_port=4443): + self._http_port = http_port + self._https_port = https_port + self._webui_port = webui_port + self._docker: Dict[str, any] = { + 'container': None + } + + def setup_ncp(self, instance_type, version): + if instance_type == 'docker': + container_name = 'nextcloudpi-' + ''.join(random.choice(string.digits) for i in range(6)) + image = version if ':' in version else f"ownyourbits/nextcloudpi:{version}" + d_run = subprocess.run([ + 'docker', 'run', '-d', + '-p', f'127.0.0.1:{self._http_port}:80', + '-p', f'127.0.0.1:{self._https_port}:443', + '-p', f'127.0.0.1:{self._webui_port}:4443', + '--name', container_name, + image, 'localhost' + ], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if d_run.returncode != 0: + raise FatalError(f"Failed to start test instance: \n'{d_run.stderr}'") + + self._docker['container'] = container_name + logger.info("Waiting for container startup") + time.sleep(20) + self.ncp_is_running() + + else: + raise FatalError(f"Invalid instance type '{instance_type}'") + + def ncp_is_running(self): + + d_run = subprocess.run(['docker', 'container', 'ls', '--filter', f'name={self._docker["container"]}'], + stdout=PIPE, stderr=PIPE) + re_string = r'.* Up.* ' + self._docker['container'] + r'.*' + docker_success_re = re.compile(re_string, flags=re.DOTALL) + + if not docker_success_re.match(d_run.stdout.decode('utf-8')): + raise AssertionError(f"Failed to start test instance: 'Container status is not 'Up'") + + def destroy_ncp(self): + # time.sleep(60) + if self._docker['container'] is None: + raise Error("Container not running") + + _ = subprocess.run(['docker', 'stop', self._docker['container']]) + try: + self.ncp_is_running() + raise FatalError("Failed to stop instance - ncp is still running") + except AssertionError: + pass + finally: + d_run = subprocess.run(['docker', 'rm', self._docker['container']]) + err = None + if d_run.returncode != 0: + err = FatalError(f"Failed to delete container {self._docker['container']}") + self._docker['container'] = None + if err is not None: + raise FatalError(err) + + def create_backup_in(self, target, *args): + logger.info(f"Creating backup in {target}...") + self.ncp_is_running() + backup_cfg = { + "id": "nc-backup", + "name": "nc-backup", + "title": "nc-backup", + "description": "Backup this NC instance to a file", + "info": "This will always include the current Nextcloud directory and the Database.\nYou can choose to include or exclude NC-data.", + "infotitle": "", + "params": [ + { + "id": "DESTDIR", + "name": "Destination directory", + "value": target, + "suggest": "/media/USBdrive/ncp-backups" + }, + { + "id": "INCLUDEDATA", + "name": "Include data", + "value": "no" if 'dataless' in args else "yes", + "type": "bool" + }, + { + "id": "COMPRESS", + "name": "Compress", + "value": "yes" if 'compressed' in args else "no", + "type": "bool" + }, + { + "id": "BACKUPLIMIT", + "name": "Number of backups to keep", + "value": "99", + "suggest": "4" + } + ] + } + temp_dir = tempfile.TemporaryDirectory() + tmp_file_path = os.path.join(temp_dir.name, 'nc-backup.cfg') + with open(tmp_file_path, 'w') as f: + json.dump(backup_cfg, f) + + backup_config_path = '/data/ncp/nc-backup.cfg' + d_run = self.run_on_ncp('rm', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = subprocess.run(['docker', 'cp', tmp_file_path, f'{self._docker["container"]}:{backup_config_path}']) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + temp_dir.cleanup() + + d_run = self.run_on_ncp('chown', 'root:www-data', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = self.run_on_ncp('chmod', '660', backup_config_path) + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"Unexpected error: {d_run.stderr}") + + d_run = self.run_on_ncp('bash', '-c', "set -x " + "&& source /usr/local/etc/library.sh " + "&& echo \"ncp library loaded.\" " + "&& run_app nc-backup") + logger.info(d_run.stdout) + if d_run.returncode != 0: + raise AssertionError(f"An error occurred while creating backup: \n{d_run.stderr}") + + def run_on_ncp(self, *command): + run_cmd = ['docker', 'exec', self._docker['container']] + run_cmd.extend(command) + logger.debug(f"Executing: '{run_cmd}'") + return subprocess.run(run_cmd, stdout=PIPE, stderr=PIPE) + + def assert_file_exists_in_archive(self, backup_path, file, silent=False): + if not silent: + logger.info(f"Checking if '{file}' exists in any archive in '{backup_path}'") + d_run = self.run_on_ncp('bash', '-c', f'compgen -G "{backup_path}"') + if d_run.returncode != 0: + raise AssertionError(f'No such backup: {backup_path}') + archive = d_run.stdout.decode('utf-8').split('\n')[0] + d_run = self.run_on_ncp('tar', '-tf', archive) + if d_run.returncode != 0: + raise AssertionError(f'Error reading archive: {archive}') + match = re.search(file, d_run.stdout.decode('utf-8')) + if match is None: + raise AssertionError(f'File not found in backup: {file}') + + def assert_file_exists_not_in_archive(self, backup_path, file): + logger.info(f"Checking that '{file}' does not exist in any archive in '{backup_path}'") + try: + self.assert_file_exists_in_archive(backup_path, file, True) + raise AssertionError(f"File unexpectedly found in backup: {file}") + except AssertionError: + pass diff --git a/tests/nc_backup_test.robot b/tests/nc_backup_test.robot new file mode 100644 index 000000000..89cd0d028 --- /dev/null +++ b/tests/nc_backup_test.robot @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +*** Settings *** +Library NcpRobotLib.py 8080 8443 4443 +Suite Setup Setup Ncp docker latest +Suite Teardown Destroy Ncp +Test Teardown Run on Ncp rm -rf /opt/ncp-backups/*.tar + +*** Test Cases *** + +Create dataless Backup + #[Documentation] Creates and validates a backup using nc-backup + Create Backup in /opt/ncp-backups dataless + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud/config/config.php + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud-sqlbkp_.*.bak + Assert file exists not in archive /opt/ncp-backups/*.tar ncdata/files/ncp + +Create Backup with data + #[Documentation] Creates and validates a backup using nc-backup + Create Backup in /opt/ncp-backups + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud/config/config.php + Assert file exists in archive /opt/ncp-backups/*.tar nextcloud-sqlbkp_.*.bak + Assert file exists in archive /opt/ncp-backups/*.tar data/ncp/files/Nextcloud.png diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..6adc4eb50 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +selenium~=4.7 +robotframework~=6.0 \ No newline at end of file From 76cc9c22e88e55544f085b57befeda1598fdbac4 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:14:43 +0100 Subject: [PATCH 12/17] Fix setting up datadir at new default location during docker build Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-datadir.sh | 18 +++++++++++------- build/docker/Dockerfile | 15 ++++++++------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index df7869b11..d2658edbd 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -21,13 +21,13 @@ install() } tmpl_opcache_dir() { - DATADIR="$(get_nc_config_value datadirectory)" + DATADIR="$(get_nc_config_value datadirectory || find_app_param nc-datadir DATADIR)" echo -n "${DATADIR}/.opcache" #[[ $( stat -fc%d / ) == $( stat -fc%d "$DATADIR" ) ]] && echo "/tmp" || echo "${DATADIR}/.opcache" } tmpl_tmp_upload_dir() { - DATADIR="$(get_nc_config_value datadirectory)" + DATADIR="$(get_nc_config_value datadirectory || find_app_param nc-datadir DATADIR)" echo -n "${DATADIR}/tmp" } @@ -101,7 +101,7 @@ configure() ## COPY cd /var/www/nextcloud - save_maintenance_mode + [[ "$BUILD_MODE" == 1 ]] || save_maintenance_mode echo "moving data directory from ${SRCDIR} to ${BASEDIR}..." @@ -124,13 +124,17 @@ configure() chown www-data: "${DATADIR}" # datadir - sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR?}"/config/config.php - ncc config:system:set logfile --value="${DATADIR}/nextcloud.log" + ncc config:system:set datadirectory --value="${DATADIR}" \ + || sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR?}"/config/config.php + + ncc config:system:set logfile --value="${DATADIR}/nextcloud.log" \ + || sed -i "s|'logfile' =>.*|'logfile' => '${DATADIR}/nextcloud.log',|" "${NCDIR?}"/config/config.php set_ncpcfg datadir "${DATADIR}" # tmp upload dir create_tmp_upload_dir - ncc config:system:set tempdirectory --value "$DATADIR/tmp" + ncc config:system:set tempdirectory --value "$DATADIR/tmp" \ + || sed -i "s|'tempdirectory' =>.*|'tempdirectory' => '${DATADIR}/tmp',|" "${NCDIR?}"/config/config.php sed -i "s|^;\?upload_tmp_dir =.*$|uploadtmp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER?}"/cli/php.ini sed -i "s|^;\?upload_tmp_dir =.*$|upload_tmp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER}"/fpm/php.ini sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = ${DATADIR}/tmp|" /etc/php/"${PHPVER}"/fpm/php.ini @@ -143,7 +147,7 @@ configure() [[ -f /etc/fail2ban/jail.local ]] && \ sed -i "s|logpath =.*nextcloud.log|logpath = ${DATADIR}/nextcloud.log|" /etc/fail2ban/jail.local - restore_maintenance_mode + [[ "$BUILD_MODE" == 1 ]] || restore_maintenance_mode ( . "${BINDIR?}/SYSTEM/metrics.sh" diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 7d9351a08..2d7a6795f 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -144,7 +144,7 @@ COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/ COPY build/docker /tmp/ncp-build/build/docker/ COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/ COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg -COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg +COPY etc/ncp-config.d/nc-datadir.cfg /usr/local/etc/ncp-config.d/nc-datadir-copy.cfg RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ @@ -168,13 +168,14 @@ install_app ncp.sh; \ mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \ run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \ -echo 'Moving data directory to a more sensible location' \ -mkdir -p /opt/ncdata \ -set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata' \ -run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh \ -set_app_param nc-datadir.sh DATADIR '/data/ncdata' \ +echo 'Moving data directory to a more sensible location'; \ +mv /usr/local/etc/ncp-config.d/nc-datadir-copy.cfg /usr/local/etc/ncp-config.d/nc-datadir.cfg; \ +mkdir -p /data-ro/ncdata; \ +set_app_param nc-datadir.sh DATADIR '/data-ro/ncdata'; \ +BUILD_MODE=1 DISABLE_FS_CHECK=1 run_app_unsafe bin/ncp/CONFIG/nc-datadir.sh; \ +set_app_param nc-datadir.sh DATADIR '/data/ncdata'; \ # Remove nc-datadir.cfg as it's disabled in docker -rm /usr/local/etc/nc-datadir.cfg \ +rm /usr/local/etc/ncp-config.d/nc-datadir.cfg; \ sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \ # fix default paths From ac848633a5b0c7a51c194d5080529de0046f8076 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:16:51 +0100 Subject: [PATCH 13/17] docker-build.yml: Fix docker update-test Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 3205c9a4d..2b9a36875 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -130,7 +130,7 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker buildx build --platform "$platform" . -t ncp-testing -f Dockerfile + docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile -o type=image,name=ncp-testing . popd rm -r "$tmpdir" docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost From f30221c18f265095f8f0f26c719ee9eeb5499657 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 21:16:58 +0100 Subject: [PATCH 14/17] ncp-restore: Don't assume data directory inside nc directory on docker Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-restore | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/ncp-restore b/bin/ncp-restore index 9b63df43d..9f78459e6 100755 --- a/bin/ncp-restore +++ b/bin/ncp-restore @@ -62,8 +62,6 @@ then target="$NCDIR/data" [[ -d "$target" ]] && target="$NCDIR/.data_$(date)" mv -T "$DATA_BKP_DIR" "$target" -else - echo "Dataless backup detected - not restoring data" fi # update NC database password to this instance @@ -93,10 +91,9 @@ mysql -u root nextcloud < "$TMPDIR"/nextcloud-sqlbkp_*.bak || { echo "Error res ## RESTORE DATADIR -if is_docker; then +DATADIR="$(get_nc_config_value datadirectory)" +if is_docker && [[ "$DATADIR" == "/var/www/nextcloud/data" ]]; then DATADIR=/data/nextcloud/data -else - DATADIR="$(get_nc_config_value datadirectory)" fi [[ "$DATADIR" == "" ]] && { echo "Error reading data directory"; exit 1; } From 67b090b00fb03fe5ed816fec6dc65450884d4ba1 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 4 Jan 2023 22:24:48 +0100 Subject: [PATCH 15/17] nc-datadir.sh: Prevent btrfs commands from being executed on docker Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-datadir.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ncp/CONFIG/nc-datadir.sh b/bin/ncp/CONFIG/nc-datadir.sh index d2658edbd..aa4c99e5e 100644 --- a/bin/ncp/CONFIG/nc-datadir.sh +++ b/bin/ncp/CONFIG/nc-datadir.sh @@ -106,7 +106,7 @@ configure() echo "moving data directory from ${SRCDIR} to ${BASEDIR}..." # use subvolumes, if BTRFS - [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]] && { + [[ "$(stat -fc%T "${BASEDIR}")" == "btrfs" ]] && ! is_docker && { echo "BTRFS filesystem detected" rmdir "${BASEDIR}" btrfs subvolume create "${BASEDIR}" From 9c6297d8f6fb76c3e98ea9eebc6a49b6b6cfd2da Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 02:19:32 +0100 Subject: [PATCH 16/17] 000ncp: Install templates during docker startup - docker-build.yml: Fix update-test job Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 4 ++-- build/docker/nextcloudpi/000ncp | 8 +++++++- etc/library.sh | 3 +++ etc/ncp-templates/php/opcache.ini.sh | 6 +++++- etc/ncp-templates/php/pool.d.www.conf.sh | 2 +- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 2b9a36875..d9d921673 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -130,7 +130,7 @@ jobs: RUN echo 'Mutex posixsem' >> /etc/apache2/mods-available/ssl.conf EOF - docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile -o type=image,name=ncp-testing . + docker buildx build --platform "$platform" -t ncp-testing -f Dockerfile --load . popd rm -r "$tmpdir" docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ncp-testing localhost @@ -201,7 +201,7 @@ jobs: for attempt in {1..3} do echo ":: System Tests (attempt $attempt/3) ::" - python system_tests.py --no-ping --non-interactive || awk "{ print \"${LOG_TEST} \" \$0 }" + python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" sleep 12 diff --git a/build/docker/nextcloudpi/000ncp b/build/docker/nextcloudpi/000ncp index ac0983518..0bd6b8c3a 100755 --- a/build/docker/nextcloudpi/000ncp +++ b/build/docker/nextcloudpi/000ncp @@ -21,7 +21,13 @@ persistent_cfg /etc/cron.weekly } persistent_cfg /usr/local/bin /data/bin +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" +install_template "php/90-ncp.ini.sh" "/etc/php/${PHPVER}/fpm/conf.d/90-ncp.ini" +install_template "php/pool.d.www.conf.sh" "/etc/php/${PHPVER}/fpm/pool.d/www.conf" +#install_template "mysql/91-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/91-ncp.cnf" +install_template "ncp-metrics.cfg.sh" "/usr/local/etc/ncp-metrics.cfg" + touch /var/log/ncp.log -tail -f -n0 /data/nextcloud/data/nextcloud.log & +{ tail -f -n0 /data/ncdata/data/nextcloud.log || tail -f -n0 /data/nextcloud/data/nextcloud.log; } & exit 0 diff --git a/etc/library.sh b/etc/library.sh index 80327d56a..46d5265db 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -216,6 +216,9 @@ install_template() { local template="${1?}" local target="${2?}" local bkp="$(mktemp)" + + echo "Installing template '$template'..." + mkdir -p "$(dirname "$target")" [[ -f "$target" ]] && cp -a "$target" "$bkp" { diff --git a/etc/ncp-templates/php/opcache.ini.sh b/etc/ncp-templates/php/opcache.ini.sh index 7bbef81b9..d92badb2a 100644 --- a/etc/ncp-templates/php/opcache.ini.sh +++ b/etc/ncp-templates/php/opcache.ini.sh @@ -5,11 +5,15 @@ source /usr/local/etc/library.sh PHPVER="${PHPVER?ERROR: PHPVER variable unset!}" -if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] +if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] && ! is_docker then echo "INFO: Restoring template to default settings" >&2 TMP_DIR="/tmp" +elif is_docker +then + DATADIR="$(get_nc_config_value datadirectory || echo '/data/ncdata/data')" + TMP_DIR="$DATADIR/.opcache" else TMP_DIR="$(source "${BINDIR}/CONFIG/nc-datadir.sh"; tmpl_opcache_dir)" fi diff --git a/etc/ncp-templates/php/pool.d.www.conf.sh b/etc/ncp-templates/php/pool.d.www.conf.sh index 451b5afff..c4f8f3f59 100644 --- a/etc/ncp-templates/php/pool.d.www.conf.sh +++ b/etc/ncp-templates/php/pool.d.www.conf.sh @@ -5,7 +5,7 @@ source /usr/local/etc/library.sh PHPVER="${PHPVER?ERROR: PHPVER variable unset!}" -if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-datadir.sh" ]] +if [[ "$1" == "--defaults" ]] || ! [[ -f "${BINDIR}/CONFIG/nc-limits.sh" ]] then echo "INFO: Restoring template to default settings" >&2 From deae8858037e4b4c38cb39e516f939043b53def9 Mon Sep 17 00:00:00 2001 From: Tobias K <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 5 Jan 2023 16:14:54 +0100 Subject: [PATCH 17/17] Fix release pipeline for lxd and docker - build-docker.yml: Increase retries for integration tests to 5 - build-lxd.yml: Fix empty output variables Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 26 +++++++++++++------------- .github/workflows/build-lxd.yml | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d9d921673..a3d228516 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -158,13 +158,13 @@ jobs: [[ "${ARCH?}" == "x86" ]] || { sleep 30; cmd+=(--timeout 300); } success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Activation Tests (attempt $attempt/3) ::" + echo ":: Activation Tests (attempt $attempt/5) ::" "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true - echo -e "${LOG_CICD} Activation test (attempt $attempt/3) failed!" + echo -e "${LOG_CICD} Activation test (attempt $attempt/5) failed!" docker exec nextcloudpi bash /usr/local/bin/ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true cat /proc/sys/fs/binfmt_misc/qemu-aarch64 |& awk "{ print \"${LOG_DIAG} (qemu-aarch64) \" \$0 }" || true cat /proc/sys/fs/binfmt_misc/qemu-arm |& awk "{ print \"${LOG_DIAG} (qemu-arm) \" \$0 }" || true @@ -193,14 +193,14 @@ jobs: echo -e "${LOG_DCKR} ==========" docker logs -f nextcloudpi |& awk "{ print \"${LOG_DCKR} \" \$0 }" & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk "{ print \"${LOG_NCP} \" \$0 }" & - [[ "${ARCH?}" == "x86" ]] || sleep 30 + [[ "${ARCH?}" == "x86" ]] || sleep 120 sleep 20 success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: System Tests (attempt $attempt/3) ::" + echo ":: System Tests (attempt $attempt/5) ::" python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" @@ -219,7 +219,7 @@ jobs: success=false for attempt in {1..3} do - echo ":: Nextcloud Tests (attempt $attempt/3) ::" + echo ":: Nextcloud Tests (attempt $attempt/5) ::" python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true @@ -293,9 +293,9 @@ jobs: [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Activation Tests (attempt $attempt/3) ::" + echo ":: Activation Tests (attempt $attempt/5) ::" "${cmd[@]}" |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true @@ -314,9 +314,9 @@ jobs: echo -e "${LOG_CICD} Activation test successful" success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: System Tests (attempt $attempt/3) ::" + echo ":: System Tests (attempt $attempt/5) ::" python system_tests.py --no-ping --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { echo -e "${LOG_CICD} System test (attempt $attempt) failed!" @@ -333,9 +333,9 @@ jobs: echo -e "${LOG_CICD} System test successful" success=false - for attempt in {1..3} + for attempt in {1..5} do - echo ":: Nextcloud Tests (attempt $attempt/3) ::" + echo ":: Nextcloud Tests (attempt $attempt/5) ::" python nextcloud_tests.py --no-gui localhost 8443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" [[ ${PIPESTATUS[0]} -eq 0 ]] || { tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index ef1ccab39..27a895598 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -7,9 +7,9 @@ on: type: string outputs: artifact_name: - value: "${{ jobs.build.outputs.artifact_name }}" + value: "${{ jobs.build-current.outputs.artifact_name }}" artifact_file: - value: "${{ jobs.build.outputs.artifact_file }}" + value: "${{ jobs.build-current.outputs.artifact_file }}" push: branches: - "**"